[FFmpeg-devel] [PATCH 1/9] dnxhdenc: fix scan used for bitstream generation

[Christophe Gisquet]

The functions related to bitstream reading must use the natural zigzag
order, and not the one permuted for use in the iDCT. This currently
results in bitstreams with 2 issues:
- AC coefficients are encoded in an unexpected order;
- Incorrect weights are applied in the (de)quantization.

This currently can't show up because there's no simple_idct which
requires a permutation.

With one that does (based on prores'), and using the following source:
-f lavfi -i testsrc=s=1920x1080 -vframes 1 -pix_fmt yuv422p10le
and encoding to CID 1241 (interlaced 1080p YUV422) using:
-vcodec dnxhd -flags +ildct -vb 185M

before: stddev:   55.57 PSNR: 61.43 MAXDIFF:  960
after:  stddev:    0.35 PSNR:105.45 MAXDIFF:   11
---
 libavcodec/dnxhdenc.c | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/libavcodec/dnxhdenc.c b/libavcodec/dnxhdenc.c
index 7d96cd4..fa0b1dc 100644
--- a/libavcodec/dnxhdenc.c
+++ b/libavcodec/dnxhdenc.c
@@ -213,14 +213,14 @@ static av_cold int dnxhd_init_qmat(DNXHDEncContext *ctx, int lbias, int cbias)
 
     if (ctx->cid_table->bit_depth == 8) {
         for (i = 1; i < 64; i++) {
-            int j = ctx->m.idsp.idct_permutation[ff_zigzag_direct[i]];
+            int j = ctx->m.intra_scantable.scantable[i];
             weight_matrix[j] = ctx->cid_table->luma_weight[i];
         }
         ff_convert_matrix(&ctx->m, ctx->qmatrix_l, ctx->qmatrix_l16,
                           weight_matrix, ctx->intra_quant_bias, 1,
                           ctx->m.avctx->qmax, 1);
         for (i = 1; i < 64; i++) {
-            int j = ctx->m.idsp.idct_permutation[ff_zigzag_direct[i]];
+            int j = ctx->m.intra_scantable.scantable[i];
             weight_matrix[j] = ctx->cid_table->chroma_weight[i];
         }
         ff_convert_matrix(&ctx->m, ctx->qmatrix_c, ctx->qmatrix_c16,
@@ -241,7 +241,7 @@ static av_cold int dnxhd_init_qmat(DNXHDEncContext *ctx, int lbias, int cbias)
         // 10-bit
         for (qscale = 1; qscale <= ctx->m.avctx->qmax; qscale++) {
             for (i = 1; i < 64; i++) {
-                int j = ctx->m.idsp.idct_permutation[ff_zigzag_direct[i]];
+                int j = ctx->m.intra_scantable.scantable[i];
 
                 /* The quantization formula from the VC-3 standard is:
                  * quantized = sign(block[i]) * floor(abs(block[i]/s) * p /
@@ -464,6 +464,7 @@ static av_always_inline
 void dnxhd_encode_block(DNXHDEncContext *ctx, int16_t *block,
                         int last_index, int n)
 {
+    const uint8_t *scan = ctx->m.intra_scantable.scantable;
     int last_non_zero = 0;
     int slevel, i, j;
 
@@ -471,7 +472,7 @@ void dnxhd_encode_block(DNXHDEncContext *ctx, int16_t *block,
     ctx->m.last_dc[n] = block[0];
 
     for (i = 1; i <= last_index; i++) {
-        j = ctx->m.intra_scantable.permutated[i];
+        j = scan[i];
         slevel = block[j];
         if (slevel) {
             int run_level = i - last_non_zero - 1;
@@ -542,11 +543,12 @@ static av_always_inline int dnxhd_ssd_block(int16_t *qblock, int16_t *block)
 static av_always_inline
 int dnxhd_calc_ac_bits(DNXHDEncContext *ctx, int16_t *block, int last_index)
 {
+    const uint8_t *scan = ctx->m.intra_scantable.scantable;
     int last_non_zero = 0;
     int bits = 0;
     int i, j, level;
     for (i = 1; i <= last_index; i++) {
-        j = ctx->m.intra_scantable.permutated[i];
+        j = scan[i];
         level = block[j];
         if (level) {
             int run_level = i - last_non_zero - 1;
-- 
2.6.0