[FFmpeg-devel] [PATCH] lavc/vaapi_encode: add support for AVC Trellis

[Linjie Fu]

Add support for VAAPI AVC Trellis Quantization with limitation:
    - VA-API version >= (1, 0, 0)

Use option "-trellis off/I/P/B" to disable or enable Trellis
quantization for I/P/B frames.

Signed-off-by: Linjie Fu <linjie.fu at intel.com>
---
 libavcodec/vaapi_encode.c      | 44 ++++++++++++++++++++++++++++++++++
 libavcodec/vaapi_encode.h      | 14 +++++++++--
 libavcodec/vaapi_encode_h264.c |  9 +++++++
 3 files changed, 65 insertions(+), 2 deletions(-)

diff --git a/libavcodec/vaapi_encode.c b/libavcodec/vaapi_encode.c
index eda8a36299..971c661a7d 100644
--- a/libavcodec/vaapi_encode.c
+++ b/libavcodec/vaapi_encode.c
@@ -1442,6 +1442,44 @@ static av_cold int vaapi_encode_init_rate_control(AVCodecContext *avctx)
     return 0;
 }
 
+static av_cold int vaapi_encode_init_quantization(AVCodecContext *avctx)
+{
+#if VA_CHECK_VERSION(1, 0, 0)
+    VAAPIEncodeContext *ctx = avctx->priv_data;
+    VAStatus vas;
+    VAConfigAttrib attr = { VAConfigAttribEncQuantization };
+    int trellis = ctx->trellis;
+
+    vas = vaGetConfigAttributes(ctx->hwctx->display,
+                                ctx->va_profile,
+                                ctx->va_entrypoint,
+                                &attr, 1);
+    if (vas != VA_STATUS_SUCCESS) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to query quantization "
+               "config attribute: %d (%s).\n", vas, vaErrorStr(vas));
+        return AVERROR_EXTERNAL;
+    }
+
+    if (attr.value == VA_ENC_QUANTIZATION_NONE) {
+        av_log(avctx, AV_LOG_WARNING, "Trellis Quantization attribute is not "
+                "supported: will use default quantization.\n");
+    } else if (attr.value == VA_ENC_QUANTIZATION_TRELLIS_SUPPORTED){
+        av_log(avctx, AV_LOG_VERBOSE, "Quantization Trellis supported.\n");
+
+        ctx->quantization_params.misc.type = VAEncMiscParameterTypeQuantization;
+        ctx->quantization_params.quantization.quantization_flags.value = trellis;
+
+        vaapi_encode_add_global_param(avctx, &ctx->quantization_params.misc,
+                                      sizeof(ctx->quantization_params));
+    }
+#else
+    av_log(avctx, AV_LOG_WARNING, "The encode quantization option (Trellis) is "
+           "not supported with this VAAPI version.\n");
+#endif
+
+    return 0;
+}
+
 static av_cold int vaapi_encode_init_gop_structure(AVCodecContext *avctx)
 {
     VAAPIEncodeContext *ctx = avctx->priv_data;
@@ -1888,6 +1926,12 @@ av_cold int ff_vaapi_encode_init(AVCodecContext *avctx)
     if (err < 0)
         goto fail;
 
+    if (ctx->trellis) {
+        err = vaapi_encode_init_quantization(avctx);
+        if (err < 0)
+            goto fail;
+    }
+
     if (avctx->compression_level >= 0) {
         err = vaapi_encode_init_quality(avctx);
         if (err < 0)
diff --git a/libavcodec/vaapi_encode.h b/libavcodec/vaapi_encode.h
index 965fe65c0b..52da90d1f6 100644
--- a/libavcodec/vaapi_encode.h
+++ b/libavcodec/vaapi_encode.h
@@ -37,7 +37,7 @@ struct VAAPIEncodePicture;
 
 enum {
     MAX_CONFIG_ATTRIBUTES  = 4,
-    MAX_GLOBAL_PARAMS      = 4,
+    MAX_GLOBAL_PARAMS      = 5,
     MAX_PICTURE_REFERENCES = 2,
     MAX_REORDER_DELAY      = 16,
     MAX_PARAM_BUFFER_SIZE  = 1024,
@@ -150,6 +150,9 @@ typedef struct VAAPIEncodeContext {
     // Packed headers which will actually be sent.
     unsigned int    va_packed_headers;
 
+    // Quantization mode
+    int trellis;
+
     // Configuration attributes to use when creating va_config.
     VAConfigAttrib  config_attributes[MAX_CONFIG_ATTRIBUTES];
     int          nb_config_attributes;
@@ -197,7 +200,14 @@ typedef struct VAAPIEncodeContext {
         VAEncMiscParameterBufferQualityLevel quality;
     } quality_params;
 #endif
-
+#if VA_CHECK_VERSION(1, 0, 0)
+    // Packed memory aligment to point uint32_t data[]
+    // in misc to the uint64_t quantization.
+    struct {
+        VAEncMiscParameterBuffer misc;
+        VAEncMiscParameterQuantization quantization;
+    } __attribute__((packed)) quantization_params;
+#endif
     // Per-sequence parameter structure (VAEncSequenceParameterBuffer*).
     void           *codec_sequence_params;
 
diff --git a/libavcodec/vaapi_encode_h264.c b/libavcodec/vaapi_encode_h264.c
index f9402992b8..4d411891fe 100644
--- a/libavcodec/vaapi_encode_h264.c
+++ b/libavcodec/vaapi_encode_h264.c
@@ -58,6 +58,7 @@ typedef struct VAAPIEncodeH264Context {
     int sei;
     int profile;
     int level;
+    int trellis;
 
     // Derived settings.
     int mb_width;
@@ -981,6 +982,8 @@ static av_cold int vaapi_encode_h264_init(AVCodecContext *avctx)
 
     ctx->slice_block_height = ctx->slice_block_width = 16;
 
+    ctx->trellis = priv->trellis;
+
     return ff_vaapi_encode_init(avctx);
 }
 
@@ -1009,6 +1012,12 @@ static const AVOption vaapi_encode_h264_options[] = {
         { "cabac", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, INT_MIN, INT_MAX, FLAGS, "coder" },
         { "vlc",   NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, INT_MIN, INT_MAX, FLAGS, "coder" },
         { "ac",    NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, INT_MIN, INT_MAX, FLAGS, "coder" },
+    { "trellis", "Trellis Quantization",
+      OFFSET(trellis), AV_OPT_TYPE_FLAGS, { .i64 = 0}, 0, INT_MAX, FLAGS, "trellis"},
+        { "off",   NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, INT_MIN, INT_MAX, FLAGS, "trellis"},
+        { "I",     NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 2 }, INT_MIN, INT_MAX, FLAGS, "trellis"},
+        { "P",     NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 4 }, INT_MIN, INT_MAX, FLAGS, "trellis"},
+        { "B",     NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 8 }, INT_MIN, INT_MAX, FLAGS, "trellis"},
 
     { "aud", "Include AUD",
       OFFSET(aud), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS },
-- 
2.17.1