[FFmpeg-cvslog] avcodec/prores_ks: Fix luma quantization if q >= MAX_STORED_Q

[Alex Mogurenko]

ffmpeg | branch: release/4.1 | Alex Mogurenko <alex at mogurenko.com> | Fri Dec 28 22:30:08 2018 +0200| [7e204f72607c95a7d7f791e085ba48882e778a9b] | committer: Michael Niedermayer

avcodec/prores_ks: Fix luma quantization if q >= MAX_STORED_Q

The problem occurs in slice quant estimation and slice encoding:

If the slice quant is larger than  MAX_STORED_Q we don't use pre-calculated
quant matrices, but generate a new one, but both qmat and qmat_chroma both
point to the same table, so the luma table ends up having chroma table
values.

Add custom_chroma_q the same way as custom_q.

Signed-off-by: Derek Buitenhuis <derek.buitenhuis at gmail.com>
(cherry picked from commit e4788ae31b2e9af45d11f4bf4498c075dcc25a6c)
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=7e204f72607c95a7d7f791e085ba48882e778a9b
---

 libavcodec/proresenc_kostya.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/libavcodec/proresenc_kostya.c b/libavcodec/proresenc_kostya.c
index 81f3865ea6..123f6c1f5a 100644
--- a/libavcodec/proresenc_kostya.c
+++ b/libavcodec/proresenc_kostya.c
@@ -221,6 +221,7 @@ typedef struct ProresThreadData {
     DECLARE_ALIGNED(16, int16_t, blocks)[MAX_PLANES][64 * 4 * MAX_MBS_PER_SLICE];
     DECLARE_ALIGNED(16, uint16_t, emu_buf)[16 * 16];
     int16_t custom_q[64];
+    int16_t custom_chroma_q[64];
     struct TrellisNode *nodes;
 } ProresThreadData;
 
@@ -231,6 +232,7 @@ typedef struct ProresContext {
     int16_t quants[MAX_STORED_Q][64];
     int16_t quants_chroma[MAX_STORED_Q][64];
     int16_t custom_q[64];
+    int16_t custom_chroma_q[64];
     const uint8_t *quant_mat;
     const uint8_t *quant_chroma_mat;
     const uint8_t *scantable;
@@ -573,7 +575,7 @@ static int encode_slice(AVCodecContext *avctx, const AVFrame *pic,
         qmat_chroma = ctx->quants_chroma[quant];
     } else {
         qmat = ctx->custom_q;
-        qmat_chroma = ctx->custom_q;
+        qmat_chroma = ctx->custom_chroma_q;
         for (i = 0; i < 64; i++) {
             qmat[i] = ctx->quant_mat[i] * quant;
             qmat_chroma[i] = ctx->quant_chroma_mat[i] * quant;
@@ -901,7 +903,7 @@ static int find_slice_quant(AVCodecContext *avctx,
                 qmat_chroma = ctx->quants_chroma[q];
             } else {
                 qmat = td->custom_q;
-                qmat_chroma = td->custom_q;
+                qmat_chroma = td->custom_chroma_q;
                 for (i = 0; i < 64; i++) {
                     qmat[i] = ctx->quant_mat[i] * q;
                     qmat_chroma[i] = ctx->quant_chroma_mat[i] * q;