[FFmpeg-devel] [PATCH 3/3] lavc/vaapi_encode_h265: enable multi-reference frames.
Jun Zhao
mypopydev at gmail.com
Wed Nov 8 10:20:51 EET 2017
-------------- next part --------------
From f7e5e56c5a64d0414a7e5fceb22cf9d962e09dc1 Mon Sep 17 00:00:00 2001
From: Jun Zhao <jun.zhao at intel.com>
Date: Tue, 7 Nov 2017 14:33:39 +0800
Subject: [PATCH 3/3] lavc/vaapi_encode_h265: enable multi-reference frames.
Enable the multi-reference frames and respect "refs" option
in hevc_vaapi encoder.
Signed-off-by: Jun Zhao <jun.zhao at intel.com>
Signed-off-by: Wang, Yi A <yi.a.wang at intel.com>
---
libavcodec/vaapi_encode_h265.c | 499 ++++++++++++++++++++++++++++++++++++-----
1 file changed, 449 insertions(+), 50 deletions(-)
diff --git a/libavcodec/vaapi_encode_h265.c b/libavcodec/vaapi_encode_h265.c
index 3ae92a7a09..85350380ca 100644
--- a/libavcodec/vaapi_encode_h265.c
+++ b/libavcodec/vaapi_encode_h265.c
@@ -34,6 +34,8 @@
#include "vaapi_encode.h"
+static const char *picture_type_name[] = { "IDR", "I", "P", "B" };
+
typedef struct VAAPIEncodeH265Context {
unsigned int ctu_width;
unsigned int ctu_height;
@@ -58,6 +60,11 @@ typedef struct VAAPIEncodeH265Context {
CodedBitstreamContext *cbc;
CodedBitstreamFragment current_access_unit;
int aud_needed;
+
+ // reference frames param
+ struct VAAPIEncodePicture *references[MAX_PICTURE_REFERENCES];
+ int ref_nr;
+ int max_ref_nr;
} VAAPIEncodeH265Context;
typedef struct VAAPIEncodeH265Options {
@@ -224,7 +231,7 @@ static int vaapi_encode_h265_init_sequence_params(AVCodecContext *avctx)
vps->profile_tier_level.general_profile_compatibility_flag[avctx->profile & 31] = 1;
vps->vps_sub_layer_ordering_info_present_flag = 0;
- vps->vps_max_dec_pic_buffering_minus1[0] = (ctx->b_per_p > 0) + 1;
+ vps->vps_max_dec_pic_buffering_minus1[0] = priv->max_ref_nr;
vps->vps_max_num_reorder_pics[0] = (ctx->b_per_p > 0);
vps->vps_max_latency_increase_plus1[0] = 0;
@@ -677,59 +684,44 @@ static int vaapi_encode_h265_init_slice_params(AVCodecContext *avctx,
if (pic->type != PICTURE_TYPE_IDR) {
H265RawSTRefPicSet *rps;
- VAAPIEncodePicture *st;
- int used;
sh->short_term_ref_pic_set_sps_flag = 0;
rps = &sh->short_term_ref_pic_set;
memset(rps, 0, sizeof(*rps));
- for (st = ctx->pic_start; st; st = st->next) {
- if (st->encode_order >= pic->encode_order) {
- // Not yet in DPB.
- continue;
+ if (pic->type != PICTURE_TYPE_B) {
+ rps->num_negative_pics = pic->nb_refs;
+ for (i = 0 ; i < rps->num_negative_pics; i++) {
+ rps->used_by_curr_pic_s0_flag[i] = 1;
+ if (i == 0) {
+ rps->delta_poc_s0_minus1[i] = priv->pic_order_cnt -
+ vpic->reference_frames[pic->nb_refs - 1 - i].pic_order_cnt - 1;
+ } else {
+ rps->delta_poc_s0_minus1[i] = vpic->reference_frames[pic->nb_refs - i].pic_order_cnt -
+ vpic->reference_frames[pic->nb_refs - 1 - i].pic_order_cnt - 1;
+ }
}
- used = 0;
- for (i = 0; i < pic->nb_refs; i++) {
- if (pic->refs[i] == st)
- used = 1;
+ } else {
+ rps->num_positive_pics = 1;
+ for (i = 0 ; i < rps->num_positive_pics; i++) {
+ rps->used_by_curr_pic_s1_flag[i] = 1;
+ rps->delta_poc_s1_minus1[i] =
+ vpic->reference_frames[pic->nb_refs- 1 - i].pic_order_cnt - priv->pic_order_cnt - 1;
}
- if (!used) {
- // Usually each picture always uses all of the others in the
- // DPB as references. The one case we have to treat here is
- // a non-IDR IRAP picture, which may need to hold unused
- // references across itself to be used for the decoding of
- // following RASL pictures. This looks for such an RASL
- // picture, and keeps the reference if there is one.
- VAAPIEncodePicture *rp;
- for (rp = ctx->pic_start; rp; rp = rp->next) {
- if (rp->encode_order < pic->encode_order)
- continue;
- if (rp->type != PICTURE_TYPE_B)
- continue;
- if (rp->refs[0] == st && rp->refs[1] == pic)
- break;
+ rps->num_negative_pics = pic->nb_refs - 1 ;
+ for (i = 0 ; i < rps->num_negative_pics; i++) {
+ rps->used_by_curr_pic_s0_flag[i] = 1;
+ if (i == 0) {
+ rps->delta_poc_s0_minus1[i] =
+ priv->pic_order_cnt - vpic->reference_frames[pic->nb_refs - 2 - i].pic_order_cnt - 1;
+ } else {
+ rps->delta_poc_s0_minus1[i] =
+ vpic->reference_frames[pic->nb_refs - 1 - i].pic_order_cnt -
+ vpic->reference_frames[pic->nb_refs - 2 - i].pic_order_cnt - 1;
}
- if (!rp)
- continue;
- }
- // This only works for one instance of each (delta_poc_sN_minus1
- // is relative to the previous frame in the list, not relative to
- // the current frame directly).
- if (st->display_order < pic->display_order) {
- rps->delta_poc_s0_minus1[rps->num_negative_pics] =
- pic->display_order - st->display_order - 1;
- rps->used_by_curr_pic_s0_flag[rps->num_negative_pics] = used;
- ++rps->num_negative_pics;
- } else {
- rps->delta_poc_s1_minus1[rps->num_positive_pics] =
- st->display_order - pic->display_order - 1;
- rps->used_by_curr_pic_s1_flag[rps->num_positive_pics] = used;
- ++rps->num_positive_pics;
}
}
-
sh->num_long_term_sps = 0;
sh->num_long_term_pics = 0;
@@ -740,9 +732,13 @@ static int vaapi_encode_h265_init_slice_params(AVCodecContext *avctx,
sh->collocated_ref_idx = 0;
}
- sh->num_ref_idx_active_override_flag = 0;
- sh->num_ref_idx_l0_active_minus1 = pps->num_ref_idx_l0_default_active_minus1;
- sh->num_ref_idx_l1_active_minus1 = pps->num_ref_idx_l1_default_active_minus1;
+ sh->num_ref_idx_active_override_flag = 1;
+ if (pic->type == PICTURE_TYPE_P) {
+ sh->num_ref_idx_l0_active_minus1 = pic->nb_refs - 1;
+ } else if (pic->type == PICTURE_TYPE_B) {
+ sh->num_ref_idx_l0_active_minus1 = pic->nb_refs - 2;
+ sh->num_ref_idx_l1_active_minus1 = 0;
+ }
}
sh->slice_sao_luma_flag = sh->slice_sao_chroma_flag =
@@ -765,8 +761,6 @@ static int vaapi_encode_h265_init_slice_params(AVCodecContext *avctx,
.num_ref_idx_l0_active_minus1 = sh->num_ref_idx_l0_active_minus1,
.num_ref_idx_l1_active_minus1 = sh->num_ref_idx_l1_active_minus1,
- .ref_pic_list0[0] = vpic->reference_frames[0],
- .ref_pic_list1[0] = vpic->reference_frames[1],
.luma_log2_weight_denom = sh->luma_log2_weight_denom,
.delta_chroma_log2_weight_denom = sh->delta_chroma_log2_weight_denom,
@@ -800,7 +794,16 @@ static int vaapi_encode_h265_init_slice_params(AVCodecContext *avctx,
},
};
-
+ if (pic->type == PICTURE_TYPE_P) {
+ for (i = 0; i < pic->nb_refs; i++) {
+ vslice->ref_pic_list0[i] = vpic->reference_frames[pic->nb_refs - 1 - i];
+ }
+ }
+ if (pic->type == PICTURE_TYPE_B) {
+ for (i = 0; i < pic->nb_refs - 1; i++)
+ vslice->ref_pic_list0[i] = vpic->reference_frames[pic->nb_refs - 2 - i];
+ vslice->ref_pic_list1[0] = vpic->reference_frames[pic->nb_refs - 1];
+ }
return 0;
}
@@ -854,6 +857,34 @@ static av_cold int vaapi_encode_h265_configure(AVCodecContext *avctx)
av_assert0(0 && "Invalid RC mode.");
}
+ priv->max_ref_nr = avctx->refs;
+
+ if (priv->max_ref_nr > ctx->max_ref_l0 + ctx->max_ref_l1) {
+ av_log(avctx, AV_LOG_WARNING, "Warning: " \
+ "reference frame number exceeds %d" \
+ "correct to %d\n", \
+ ctx->max_ref_l0 + ctx->max_ref_l1,
+ ctx->max_ref_l0 + ctx->max_ref_l1);
+ priv->max_ref_nr = ctx->max_ref_l0 + ctx->max_ref_l1;
+ }
+ if (avctx->max_b_frames && priv->max_ref_nr < 2) {
+ av_log(avctx, AV_LOG_WARNING, "Warning: " \
+ "reference frame number is 1 but b frame encoding is setted," \
+ "correct to 2\n");
+ priv->max_ref_nr = 2;
+ }
+ if (!avctx->max_b_frames && priv->max_ref_nr > ctx->max_ref_l0) {
+ av_log(avctx, AV_LOG_WARNING, "Warning: " \
+ "no b frame, but ref_nr > max_ref_l0" \
+ "correct to %d\n", ctx->max_ref_l0);
+ priv->max_ref_nr = ctx->max_ref_l0;
+ }
+ if (priv->max_ref_nr < 1 && avctx->gop_size) {
+ av_log(avctx, AV_LOG_WARNING, "Warning: " \
+ "reference frame number is %d but gop_size > 0," \
+ "correct to 1\n", priv->max_ref_nr);
+ priv->max_ref_nr = 1;
+ }
return 0;
}
@@ -936,6 +967,374 @@ static av_cold int vaapi_encode_h265_close(AVCodecContext *avctx)
return ff_vaapi_encode_close(avctx);
}
+static void vaapi_encode_h265_add_reference (AVCodecContext *avctx,
+ VAAPIEncodePicture *pic)
+{
+ VAAPIEncodeContext *ctx = avctx->priv_data;
+ VAAPIEncodeH265Context *priv = ctx->priv_data;
+ int i;
+
+ av_assert0 (pic->type != PICTURE_TYPE_B);
+
+ if (pic->type == PICTURE_TYPE_IDR) {
+ // clear the reference frame list
+ for (i = 0 ; i < priv->ref_nr; i++) {
+ priv->references[i]->ref_count --;
+ priv->references[i] = NULL;
+ }
+ priv->ref_nr = 0;
+ }
+
+ if (priv->ref_nr == priv->max_ref_nr) {
+ // remove the oldest reference frame
+ for (i = 0 ; i < priv->ref_nr - 1; i++) {
+ priv->references[i]->ref_count --;
+ priv->references[i] = priv->references[i+1];
+ priv->references[i]->ref_count ++;
+ }
+ priv->references[priv->ref_nr-1]->ref_count--;
+ priv->ref_nr --;
+ }
+
+ priv->references[priv->ref_nr] = pic;
+ priv->references[priv->ref_nr]->ref_count ++;
+ priv->ref_nr++;
+}
+
+static int vaapi_encode_h265_get_next(AVCodecContext *avctx,
+ VAAPIEncodePicture **pic_out)
+{
+ VAAPIEncodeContext *ctx = avctx->priv_data;
+ VAAPIEncodeH265Context *priv = ctx->priv_data;
+ VAAPIEncodePicture *start, *end, *pic;
+ int i,j;
+
+ for (pic = ctx->pic_start; pic; pic = pic->next) {
+ if (pic->next)
+ av_assert0(pic->display_order < pic->next->display_order);
+ if (pic->display_order == ctx->input_order) {
+ *pic_out = pic;
+ return 0;
+ }
+ }
+
+ pic = vaapi_encode_alloc();
+ if (!pic)
+ return AVERROR(ENOMEM);
+
+ if (ctx->input_order == 0 || ctx->force_idr ||
+ ctx->gop_counter >= avctx->gop_size) {
+ pic->type = PICTURE_TYPE_IDR;
+ ctx->force_idr = 0;
+ ctx->gop_counter = 1;
+ ctx->p_counter = 0;
+ } else if (ctx->p_counter >= ctx->p_per_i) {
+ pic->type = PICTURE_TYPE_I;
+ ++ctx->gop_counter;
+ ctx->p_counter = 0;
+ } else {
+ pic->type = PICTURE_TYPE_P;
+ for (i = 0 ; i < priv->ref_nr; i++) {
+ pic->refs[i] = priv->references[i];
+ pic->refs[i]->ref_count++;
+ }
+ pic->nb_refs = priv->ref_nr;
+ ++ctx->gop_counter;
+ ++ctx->p_counter;
+ }
+ start = end = pic;
+ vaapi_encode_h265_add_reference(avctx, pic);
+
+ if (pic->type != PICTURE_TYPE_IDR) {
+ // If that was not an IDR frame, add B-frames display-before and
+ // encode-after it, but not exceeding the GOP size.
+
+ for (i = 0; i < ctx->b_per_p &&
+ ctx->gop_counter < avctx->gop_size; i++) {
+ pic = vaapi_encode_alloc();
+ if (!pic)
+ goto fail;
+
+ pic->type = PICTURE_TYPE_B;
+ for (j = 0 ; j < priv->ref_nr; j++) {
+ pic->refs[j] = priv->references[j];
+ pic->refs[j]->ref_count++;
+ }
+ pic->nb_refs = priv->ref_nr;
+ pic->next = start;
+ pic->display_order = ctx->input_order + ctx->b_per_p - i - 1;
+ pic->encode_order = pic->display_order + 1;
+ start = pic;
+
+ ++ctx->gop_counter;
+ }
+ }
+
+ if (ctx->input_order == 0) {
+ pic->display_order = 0;
+ pic->encode_order = 0;
+
+ ctx->pic_start = ctx->pic_end = pic;
+
+ } else {
+ for (i = 0, pic = start; pic; i++, pic = pic->next) {
+ pic->display_order = ctx->input_order + i;
+ if (end->type == PICTURE_TYPE_IDR)
+ pic->encode_order = ctx->input_order + i;
+ else if (pic == end)
+ pic->encode_order = ctx->input_order;
+ else
+ pic->encode_order = ctx->input_order + i + 1;
+ }
+
+ av_assert0(ctx->pic_end);
+ ctx->pic_end->next = start;
+ ctx->pic_end = end;
+ }
+ *pic_out = start;
+
+ av_log(avctx, AV_LOG_DEBUG, "Pictures:");
+ for (pic = ctx->pic_start; pic; pic = pic->next) {
+ av_log(avctx, AV_LOG_DEBUG, " %s (%"PRId64"/%"PRId64")",
+ picture_type_name[pic->type],
+ pic->display_order, pic->encode_order);
+ }
+ av_log(avctx, AV_LOG_DEBUG, "\n");
+
+ return 0;
+
+fail:
+ while (start) {
+ pic = start->next;
+ vaapi_encode_free(avctx, start);
+ start = pic;
+ }
+ return AVERROR(ENOMEM);
+}
+
+static int vaapi_encode_h265_clear_old(AVCodecContext *avctx)
+{
+ VAAPIEncodeContext *ctx = avctx->priv_data;
+ VAAPIEncodePicture *pic, *next;
+ pic = ctx->pic_start;
+ while (pic && pic->next) {
+ if (pic->encode_order > ctx->output_order)
+ break;
+
+ if (pic->ref_count == 0 && pic == ctx->pic_start) {
+ ctx->pic_start = pic->next;
+ vaapi_encode_free(avctx, pic);
+ pic = ctx->pic_start;
+ continue;
+ }
+ next = pic->next;
+
+ if (next->encode_order > ctx->output_order)
+ break;
+ if (next->ref_count == 0) {
+ pic->next = next->next;
+ vaapi_encode_free(avctx, next);
+ }
+ pic = pic->next;
+ }
+ return 0;
+}
+
+static int vaapi_encode_h265_truncate_gop(AVCodecContext *avctx)
+{
+ VAAPIEncodeContext *ctx = avctx->priv_data;
+ VAAPIEncodeH265Context *priv = ctx->priv_data;
+ VAAPIEncodePicture *pic, *last_pic, *next;
+
+ // Find the last picture we actually have input for.
+ for (pic = ctx->pic_start; pic; pic = pic->next) {
+ if (!pic->input_available)
+ break;
+ last_pic = pic;
+ }
+
+ if (pic) {
+ av_assert0(last_pic);
+
+ if (last_pic->type == PICTURE_TYPE_B) {
+ // Some fixing up is required. Change the type of this
+ // picture to P, then modify preceding B references which
+ // point beyond it to point at it instead.
+ int last_ref = last_pic->nb_refs - 1;
+
+ last_pic->type = PICTURE_TYPE_P;
+ last_pic->encode_order = last_pic->refs[last_ref]->encode_order;
+
+ for (pic = ctx->pic_start; pic != last_pic; pic = pic->next) {
+ if (pic->type == PICTURE_TYPE_B &&
+ pic->refs[last_ref] == last_pic->refs[last_ref]) {
+ if (last_pic->refs[last_ref])
+ pic->refs[last_ref]->ref_count --;
+ pic->refs[last_ref] = last_pic;
+ pic->refs[last_ref]->ref_count ++;
+ }
+ }
+
+ last_pic->nb_refs = last_pic->refs[last_ref] ? last_pic->nb_refs - 1 : last_pic->nb_refs;
+
+ if (last_pic->refs[last_ref])
+ last_pic->refs[last_ref]->ref_count--;
+ last_pic->refs[last_ref] = NULL;
+ } else {
+ // We can use the current structure (no references point
+ // beyond the end), but there are unused pics to discard.
+ }
+
+ // Discard all following pics, they will never be used.
+ for (pic = last_pic->next; pic; pic = next) {
+ int i;
+ int ref_nr = priv->ref_nr;
+ next = pic->next;
+
+ for (i = 0; i < pic->nb_refs; i++) {
+ pic->refs[i]->ref_count--;
+ }
+ for (i = 0 ; i < ref_nr; i++) {
+ if (priv->references[i] == pic) {
+ priv->references[i]->ref_count--;
+ priv->references[i] = NULL;
+ priv->ref_nr --;
+ }
+ }
+ vaapi_encode_free(avctx, pic);
+ }
+
+ last_pic->next = NULL;
+ ctx->pic_end = last_pic;
+
+ } else {
+ // Input is available for all pictures, so we don't need to
+ // mangle anything.
+ }
+
+ av_log(avctx, AV_LOG_DEBUG, "Pictures ending truncated GOP:");
+ for (pic = ctx->pic_start; pic; pic = pic->next) {
+ av_log(avctx, AV_LOG_DEBUG, " %s (%"PRId64"/%"PRId64")",
+ picture_type_name[pic->type],
+ pic->display_order, pic->encode_order);
+ }
+ av_log(avctx, AV_LOG_DEBUG, "\n");
+
+ return 0;
+}
+
+static av_cold int vaapi_encode_h265_encode(AVCodecContext *avctx,
+ AVPacket *pkt,
+ const AVFrame *input_image,
+ int *got_packet)
+{
+ VAAPIEncodeContext *ctx = avctx->priv_data;
+ VAAPIEncodePicture *pic;
+ int err;
+
+ if (input_image) {
+ av_log(avctx, AV_LOG_DEBUG, "Encode frame: %ux%u (%"PRId64").\n",
+ input_image->width, input_image->height, input_image->pts);
+
+ if (input_image->pict_type == AV_PICTURE_TYPE_I) {
+ err = vaapi_encode_h265_truncate_gop(avctx);
+ if (err < 0)
+ goto fail;
+ ctx->force_idr = 1;
+ }
+
+ err = vaapi_encode_h265_get_next(avctx, &pic);
+ if (err) {
+ av_log(avctx, AV_LOG_ERROR, "Input setup failed: %d.\n", err);
+ return err;
+ }
+
+ pic->input_image = av_frame_alloc();
+ if (!pic->input_image) {
+ err = AVERROR(ENOMEM);
+ goto fail;
+ }
+ err = av_frame_ref(pic->input_image, input_image);
+ if (err < 0)
+ goto fail;
+ pic->input_surface = (VASurfaceID)(uintptr_t)input_image->data[3];
+ pic->pts = input_image->pts;
+
+ if (ctx->input_order == 0)
+ ctx->first_pts = pic->pts;
+ if (ctx->input_order == ctx->decode_delay)
+ ctx->dts_pts_diff = pic->pts - ctx->first_pts;
+ if (ctx->output_delay > 0)
+ ctx->ts_ring[ctx->input_order % (3 * ctx->output_delay)] = pic->pts;
+
+ pic->input_available = 1;
+
+ } else {
+ if (!ctx->end_of_stream) {
+ err = vaapi_encode_h265_truncate_gop(avctx);
+ if (err < 0)
+ goto fail;
+ ctx->end_of_stream = 1;
+ }
+ }
+
+ ++ctx->input_order;
+ ++ctx->output_order;
+ av_assert0(ctx->output_order + ctx->output_delay + 1 == ctx->input_order);
+
+ for (pic = ctx->pic_start; pic; pic = pic->next)
+ if (pic->encode_order == ctx->output_order)
+ break;
+
+ // pic can be null here if we don't have a specific target in this
+ // iteration. We might still issue encodes if things can be overlapped,
+ // even though we don't intend to output anything.
+
+ err = vaapi_encode_step(avctx, pic);
+ if (err < 0) {
+ av_log(avctx, AV_LOG_ERROR, "Encode failed: %d.\n", err);
+ goto fail;
+ }
+
+ if (!pic) {
+ *got_packet = 0;
+ } else {
+ err = vaapi_encode_output(avctx, pic, pkt);
+ if (err < 0) {
+ av_log(avctx, AV_LOG_ERROR, "Output failed: %d.\n", err);
+ goto fail;
+ }
+
+ if (ctx->output_delay == 0) {
+ pkt->dts = pkt->pts;
+ } else if (ctx->output_order < ctx->decode_delay) {
+ if (ctx->ts_ring[ctx->output_order] < INT64_MIN + ctx->dts_pts_diff)
+ pkt->dts = INT64_MIN;
+ else
+ pkt->dts = ctx->ts_ring[ctx->output_order] - ctx->dts_pts_diff;
+ } else {
+ pkt->dts = ctx->ts_ring[(ctx->output_order - ctx->decode_delay) %
+ (3 * ctx->output_delay)];
+ }
+
+ *got_packet = 1;
+ }
+
+ err = vaapi_encode_h265_clear_old(avctx);
+ if (err < 0) {
+ av_log(avctx, AV_LOG_ERROR, "List clearing failed: %d.\n", err);
+ goto fail;
+ }
+
+ return 0;
+
+fail:
+ // Unclear what to clean up on failure. There are probably some things we
+ // could do usefully clean up here, but for now just leave them for uninit()
+ // to do instead.
+ return err;
+}
+
#define OFFSET(x) (offsetof(VAAPIEncodeContext, codec_options_data) + \
offsetof(VAAPIEncodeH265Options, x))
#define FLAGS (AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM)
@@ -977,7 +1376,7 @@ AVCodec ff_hevc_vaapi_encoder = {
.priv_data_size = (sizeof(VAAPIEncodeContext) +
sizeof(VAAPIEncodeH265Options)),
.init = &vaapi_encode_h265_init,
- .encode2 = &ff_vaapi_encode2,
+ .encode2 = &vaapi_encode_h265_encode,
.close = &vaapi_encode_h265_close,
.priv_class = &vaapi_encode_h265_class,
.capabilities = AV_CODEC_CAP_DELAY,
--
2.14.1
More information about the ffmpeg-devel
mailing list