[FFmpeg-devel] [PATCH] jpeg2000: fix overflow in dequantization

[Michael Niedermayer]

Fixes decoding of file generated with:
ffmpeg -f lavfi -i smptehdbars=hd720 -pix_fmt rgb48 /tmp/o.jp2

Signed-off-by: Michael Niedermayer <michaelni at gmx.at>
---
 libavcodec/j2kenc.c      |    4 ++--
 libavcodec/jpeg2000.c    |    2 +-
 libavcodec/jpeg2000dec.c |    3 ++-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c
index 48c1db8..fa15aad 100644
--- a/libavcodec/j2kenc.c
+++ b/libavcodec/j2kenc.c
@@ -802,7 +802,7 @@ static void truncpasses(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile)
                         Jpeg2000Cblk *cblk = prec->cblk + cblkno;
 
                         cblk->ninclpasses = getcut(cblk, s->lambda,
-                                (int64_t)dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 16);
+                                (int64_t)dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 15);
                     }
                 }
             }
@@ -863,7 +863,7 @@ static int encode_tile(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile, int tileno
                                 int *ptr = t1.data[y-yy0];
                                 for (x = xx0; x < xx1; x++){
                                     *ptr = (comp->i_data[(comp->coord[0][1] - comp->coord[0][0]) * y + x]);
-                                    *ptr = (int64_t)*ptr * (int64_t)(16384 * 65536 / band->i_stepsize) >> 14 - NMSEDEC_FRACBITS;
+                                    *ptr = (int64_t)*ptr * (int64_t)(16384 * 65536 / band->i_stepsize) >> 15 - NMSEDEC_FRACBITS;
                                     ptr++;
                                 }
                             }
diff --git a/libavcodec/jpeg2000.c b/libavcodec/jpeg2000.c
index f044164..cb1a64d 100644
--- a/libavcodec/jpeg2000.c
+++ b/libavcodec/jpeg2000.c
@@ -320,7 +320,7 @@ int ff_jpeg2000_init_component(Jpeg2000Component *comp,
             if (!av_codec_is_encoder(avctx->codec))
                 band->f_stepsize *= 0.5;
 
-            band->i_stepsize = band->f_stepsize * (1 << 16);
+            band->i_stepsize = band->f_stepsize * (1 << 15);
 
             /* computation of tbx_0, tbx_1, tby_0, tby_1
              * see ISO/IEC 15444-1:2002 B.5 eq. B-15 and tbl B.1
diff --git a/libavcodec/jpeg2000dec.c b/libavcodec/jpeg2000dec.c
index e2a9a95..ff611a3 100644
--- a/libavcodec/jpeg2000dec.c
+++ b/libavcodec/jpeg2000dec.c
@@ -311,6 +311,7 @@ static int get_cox(Jpeg2000DecoderContext *s, Jpeg2000CodingStyle *c)
         av_log(s->avctx, AV_LOG_WARNING, "extra cblk styles %X\n", c->cblk_style);
     }
     c->transform = bytestream2_get_byteu(&s->g); // DWT transformation type
+
     /* set integer 9/7 DWT in case of BITEXACT flag */
     if ((s->avctx->flags & CODEC_FLAG_BITEXACT) && (c->transform == FF_DWT97))
         c->transform = FF_DWT97_INT;
@@ -1024,7 +1025,7 @@ static void dequantization_int(int x, int y, Jpeg2000Cblk *cblk,
         int32_t *datap = &comp->i_data[(comp->coord[0][1] - comp->coord[0][0]) * (y + j) + x];
         int *src = t1->data[j];
         for (i = 0; i < w; ++i)
-            datap[i] = (src[i] * band->i_stepsize + (1 << 15)) >> 16;
+            datap[i] = (src[i] * band->i_stepsize + (1 << 14)) >> 15;
     }
 }
 
-- 
1.7.9.5