[FFmpeg-devel] [PATCH v3, 1/2] lavc/vaapi_encode: add support for AVC Trellis

[Linjie Fu]

Trellis quantization is an algorithm that can improve data compression
in DCT-based encoding methods. It reduces the size of some DCT
coefficients while recovering others to take their place.

Trellis quantization effectively finds the optimal quantization for
each block to maximize the PSNR relative to bitrate.

Add support for VAAPI AVC Trellis Quantization with limitation:
    - VA-API version >= (1, 5, 0)

Use option "-trellis off/I/P/B" to disable or enable Trellis
quantization for I/P/B frames.

Signed-off-by: Linjie Fu <linjie.fu at intel.com>
---
 libavcodec/vaapi_encode.c | 47 +++++++++++++++++++++++++++++++++++++++
 libavcodec/vaapi_encode.h | 19 +++++++++++++---
 2 files changed, 63 insertions(+), 3 deletions(-)

diff --git a/libavcodec/vaapi_encode.c b/libavcodec/vaapi_encode.c
index dd2a24de04..394c824069 100644
--- a/libavcodec/vaapi_encode.c
+++ b/libavcodec/vaapi_encode.c
@@ -1671,6 +1671,47 @@ rc_mode_found:
     return 0;
 }
 
+static av_cold int vaapi_encode_init_quantization(AVCodecContext *avctx)
+{
+#if VA_CHECK_VERSION(1, 5, 0)
+    VAAPIEncodeContext *ctx = avctx->priv_data;
+    VAStatus vas;
+    VAConfigAttrib attr = { VAConfigAttribEncQuantization };
+
+    vas = vaGetConfigAttributes(ctx->hwctx->display,
+                                ctx->va_profile,
+                                ctx->va_entrypoint,
+                                &attr, 1);
+    if (vas != VA_STATUS_SUCCESS) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to query quantization "
+               "config attribute: %d (%s).\n", vas, vaErrorStr(vas));
+        return AVERROR_EXTERNAL;
+    }
+
+    if (attr.value == VA_ATTRIB_NOT_SUPPORTED ||
+        attr.value == VA_ENC_QUANTIZATION_NONE) {
+        av_log(avctx, AV_LOG_WARNING, "Special Quantization attribute is not "
+                "supported: will use default quantization.\n");
+    } else if (attr.value == VA_ENC_QUANTIZATION_TRELLIS_SUPPORTED){
+        av_log(avctx, AV_LOG_VERBOSE, "Quantization Trellis supported.\n");
+
+        ctx->quantization_params = (VAEncMiscParameterQuantization) {
+            .quantization_flags.value = ctx->trellis,
+        };
+
+        vaapi_encode_add_global_param(avctx,
+                                      VAEncMiscParameterTypeQuantization,
+                                      &ctx->quantization_params,
+                                      sizeof(ctx->quantization_params));
+    }
+#else
+    av_log(avctx, AV_LOG_WARNING, "The encode quantization option (Trellis) is "
+           "not supported with this VAAPI version.\n");
+#endif
+
+    return 0;
+}
+
 static av_cold int vaapi_encode_init_gop_structure(AVCodecContext *avctx)
 {
     VAAPIEncodeContext *ctx = avctx->priv_data;
@@ -2132,6 +2173,12 @@ av_cold int ff_vaapi_encode_init(AVCodecContext *avctx)
     if (err < 0)
         goto fail;
 
+    if (ctx->trellis) {
+        err = vaapi_encode_init_quantization(avctx);
+        if (err < 0)
+            goto fail;
+    }
+
     if (avctx->compression_level >= 0) {
         err = vaapi_encode_init_quality(avctx);
         if (err < 0)
diff --git a/libavcodec/vaapi_encode.h b/libavcodec/vaapi_encode.h
index eeec06036b..9b7f29a2d2 100644
--- a/libavcodec/vaapi_encode.h
+++ b/libavcodec/vaapi_encode.h
@@ -37,7 +37,7 @@ struct VAAPIEncodePicture;
 
 enum {
     MAX_CONFIG_ATTRIBUTES  = 4,
-    MAX_GLOBAL_PARAMS      = 4,
+    MAX_GLOBAL_PARAMS      = 5,
     MAX_DPB_SIZE           = 16,
     MAX_PICTURE_REFERENCES = 2,
     MAX_REORDER_DELAY      = 16,
@@ -176,6 +176,9 @@ typedef struct VAAPIEncodeContext {
     // Desired B frame reference depth.
     int             desired_b_depth;
 
+    // Quantization mode
+    int             trellis;
+
     // Explicitly set RC mode (otherwise attempt to pick from
     // available modes).
     int             explicit_rc_mode;
@@ -256,7 +259,9 @@ typedef struct VAAPIEncodeContext {
 #if VA_CHECK_VERSION(0, 36, 0)
     VAEncMiscParameterBufferQualityLevel quality_params;
 #endif
-
+#if VA_CHECK_VERSION(1, 5, 0)
+    VAEncMiscParameterQuantization quantization_params;
+#endif
     // Per-sequence parameter structure (VAEncSequenceParameterBuffer*).
     void           *codec_sequence_params;
 
@@ -418,7 +423,15 @@ int ff_vaapi_encode_close(AVCodecContext *avctx);
     { "b_depth", \
       "Maximum B-frame reference depth", \
       OFFSET(common.desired_b_depth), AV_OPT_TYPE_INT, \
-      { .i64 = 1 }, 1, INT_MAX, FLAGS }
+      { .i64 = 1 }, 1, INT_MAX, FLAGS }, \
+    { "trellis", \
+      "Trellis Quantization", \
+      OFFSET(common.trellis), AV_OPT_TYPE_FLAGS, \
+      { .i64 = 0}, 0, INT_MAX, FLAGS, "trellis"}, \
+      { "off",   NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, INT_MIN, INT_MAX, FLAGS, "trellis"}, \
+      { "I",     NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 2 }, INT_MIN, INT_MAX, FLAGS, "trellis"}, \
+      { "P",     NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 4 }, INT_MIN, INT_MAX, FLAGS, "trellis"}, \
+      { "B",     NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 8 }, INT_MIN, INT_MAX, FLAGS, "trellis"}
 
 #define VAAPI_ENCODE_RC_MODE(name, desc) \
     { #name, desc, 0, AV_OPT_TYPE_CONST, { .i64 = RC_MODE_ ## name }, \
-- 
2.17.1