[FFmpeg-devel] [PATCH, v2] lavc/vaapi_encode: add support for AVC Trellis

[Linjie Fu]

Add support for VAAPI AVC Trellis Quantization with limitation:
    - VA-API version >= (1, 0, 0)

Use option "-trellis off/I/P/B" to disable or enable Trellis
quantization for I/P/B frames.

Signed-off-by: Linjie Fu <linjie.fu at intel.com>
---
[v2]: Since nonstandard struct for VAEncMiscParameterQuantization is
fixed: https://github.com/intel/libva/issues/265
update patch based on:
http://git.ffmpeg.org/gitweb/ffmpeg.git/commit/2880a32c668023bfee4745095c885450d547ae45
 libavcodec/vaapi_encode.c      | 48 ++++++++++++++++++++++++++++++++++
 libavcodec/vaapi_encode.h      |  9 +++++--
 libavcodec/vaapi_encode_h264.c |  9 +++++++
 3 files changed, 64 insertions(+), 2 deletions(-)

diff --git a/libavcodec/vaapi_encode.c b/libavcodec/vaapi_encode.c
index dd2a24de04..fbfbe78c6b 100644
--- a/libavcodec/vaapi_encode.c
+++ b/libavcodec/vaapi_encode.c
@@ -1671,6 +1671,48 @@ rc_mode_found:
     return 0;
 }
 
+static av_cold int vaapi_encode_init_quantization(AVCodecContext *avctx)
+{
+#if VA_CHECK_VERSION(1, 0, 0)
+    VAAPIEncodeContext *ctx = avctx->priv_data;
+    VAStatus vas;
+    VAConfigAttrib attr = { VAConfigAttribEncQuantization };
+    int trellis = ctx->trellis;
+
+    vas = vaGetConfigAttributes(ctx->hwctx->display,
+                                ctx->va_profile,
+                                ctx->va_entrypoint,
+                                &attr, 1);
+    if (vas != VA_STATUS_SUCCESS) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to query quantization "
+               "config attribute: %d (%s).\n", vas, vaErrorStr(vas));
+        return AVERROR_EXTERNAL;
+    }
+
+    if (attr.value == VA_ATTRIB_NOT_SUPPORTED ||
+        attr.value == VA_ENC_QUANTIZATION_NONE) {
+        av_log(avctx, AV_LOG_WARNING, "Special Quantization attribute is not "
+                "supported: will use default quantization.\n");
+    } else if (attr.value == VA_ENC_QUANTIZATION_TRELLIS_SUPPORTED){
+        av_log(avctx, AV_LOG_VERBOSE, "Quantization Trellis supported.\n");
+
+        ctx->quantization_params = (VAEncMiscParameterQuantization) {
+            .quantization_flags.value = trellis,
+        };
+
+        vaapi_encode_add_global_param(avctx,
+                                      VAEncMiscParameterTypeQuantization,
+                                      &ctx->quantization_params,
+                                      sizeof(ctx->quantization_params));
+    }
+#else
+    av_log(avctx, AV_LOG_WARNING, "The encode quantization option (Trellis) is "
+           "not supported with this VAAPI version.\n");
+#endif
+
+    return 0;
+}
+
 static av_cold int vaapi_encode_init_gop_structure(AVCodecContext *avctx)
 {
     VAAPIEncodeContext *ctx = avctx->priv_data;
@@ -2132,6 +2174,12 @@ av_cold int ff_vaapi_encode_init(AVCodecContext *avctx)
     if (err < 0)
         goto fail;
 
+    if (ctx->trellis) {
+        err = vaapi_encode_init_quantization(avctx);
+        if (err < 0)
+            goto fail;
+    }
+
     if (avctx->compression_level >= 0) {
         err = vaapi_encode_init_quality(avctx);
         if (err < 0)
diff --git a/libavcodec/vaapi_encode.h b/libavcodec/vaapi_encode.h
index eeec06036b..b24735da59 100644
--- a/libavcodec/vaapi_encode.h
+++ b/libavcodec/vaapi_encode.h
@@ -37,7 +37,7 @@ struct VAAPIEncodePicture;
 
 enum {
     MAX_CONFIG_ATTRIBUTES  = 4,
-    MAX_GLOBAL_PARAMS      = 4,
+    MAX_GLOBAL_PARAMS      = 5,
     MAX_DPB_SIZE           = 16,
     MAX_PICTURE_REFERENCES = 2,
     MAX_REORDER_DELAY      = 16,
@@ -220,6 +220,9 @@ typedef struct VAAPIEncodeContext {
     // Packed headers which will actually be sent.
     unsigned int    va_packed_headers;
 
+    // Quantization mode
+    int trellis;
+
     // Configuration attributes to use when creating va_config.
     VAConfigAttrib  config_attributes[MAX_CONFIG_ATTRIBUTES];
     int          nb_config_attributes;
@@ -256,7 +259,9 @@ typedef struct VAAPIEncodeContext {
 #if VA_CHECK_VERSION(0, 36, 0)
     VAEncMiscParameterBufferQualityLevel quality_params;
 #endif
-
+#if VA_CHECK_VERSION(1, 0, 0)
+    VAEncMiscParameterQuantization quantization_params;
+#endif
     // Per-sequence parameter structure (VAEncSequenceParameterBuffer*).
     void           *codec_sequence_params;
 
diff --git a/libavcodec/vaapi_encode_h264.c b/libavcodec/vaapi_encode_h264.c
index d1427112ea..427fb6320e 100644
--- a/libavcodec/vaapi_encode_h264.c
+++ b/libavcodec/vaapi_encode_h264.c
@@ -72,6 +72,7 @@ typedef struct VAAPIEncodeH264Context {
     int sei;
     int profile;
     int level;
+    int trellis;
 
     // Derived settings.
     int mb_width;
@@ -1233,6 +1234,8 @@ static av_cold int vaapi_encode_h264_init(AVCodecContext *avctx)
     if (priv->qp > 0)
         ctx->explicit_qp = priv->qp;
 
+    ctx->trellis = priv->trellis;
+
     return ff_vaapi_encode_init(avctx);
 }
 
@@ -1263,6 +1266,12 @@ static const AVOption vaapi_encode_h264_options[] = {
         { "cabac", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, INT_MIN, INT_MAX, FLAGS, "coder" },
         { "vlc",   NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, INT_MIN, INT_MAX, FLAGS, "coder" },
         { "ac",    NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, INT_MIN, INT_MAX, FLAGS, "coder" },
+    { "trellis", "Trellis Quantization",
+      OFFSET(trellis), AV_OPT_TYPE_FLAGS, { .i64 = 0}, 0, INT_MAX, FLAGS, "trellis"},
+        { "off",   NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, INT_MIN, INT_MAX, FLAGS, "trellis"},
+        { "I",     NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 2 }, INT_MIN, INT_MAX, FLAGS, "trellis"},
+        { "P",     NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 4 }, INT_MIN, INT_MAX, FLAGS, "trellis"},
+        { "B",     NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 8 }, INT_MIN, INT_MAX, FLAGS, "trellis"},
 
     { "aud", "Include AUD",
       OFFSET(aud), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS },
-- 
2.17.1