[FFmpeg-cvslog] avcodec/mss34dsp: Fix multiple signed integer overflow

[Michael Niedermayer]

ffmpeg | branch: master | Michael Niedermayer <michael at niedermayer.cc> | Sun May  7 14:12:04 2017 +0200| [464c4b86ee43b7912e6f23fd3e5ba40381b4c371] | committer: Michael Niedermayer

avcodec/mss34dsp: Fix multiple signed integer overflow

Fixes: 1387/clusterfuzz-testcase-minimized-4802757766676480

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=464c4b86ee43b7912e6f23fd3e5ba40381b4c371
---

 libavcodec/mss34dsp.c | 44 ++++++++++++++++++++++----------------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/libavcodec/mss34dsp.c b/libavcodec/mss34dsp.c
index 36e69db8cc..f3405658f7 100644
--- a/libavcodec/mss34dsp.c
+++ b/libavcodec/mss34dsp.c
@@ -62,30 +62,30 @@ void ff_mss34_gen_quant_mat(uint16_t *qmat, int quality, int luma)
 }
 
 #define DCT_TEMPLATE(blk, step, SOP, shift)                         \
-    const int t0 = -39409 * blk[7 * step] -  58980 * blk[1 * step]; \
-    const int t1 =  39410 * blk[1 * step] -  58980 * blk[7 * step]; \
-    const int t2 = -33410 * blk[5 * step] - 167963 * blk[3 * step]; \
-    const int t3 =  33410 * blk[3 * step] - 167963 * blk[5 * step]; \
-    const int t4 =          blk[3 * step] +          blk[7 * step]; \
-    const int t5 =          blk[1 * step] +          blk[5 * step]; \
-    const int t6 =  77062 * t4            +  51491 * t5;            \
-    const int t7 =  77062 * t5            -  51491 * t4;            \
-    const int t8 =  35470 * blk[2 * step] -  85623 * blk[6 * step]; \
-    const int t9 =  35470 * blk[6 * step] +  85623 * blk[2 * step]; \
-    const int tA = SOP(blk[0 * step] - blk[4 * step]);              \
-    const int tB = SOP(blk[0 * step] + blk[4 * step]);              \
+    const unsigned t0 =-39409U * blk[7 * step] - 58980U * blk[1 * step]; \
+    const unsigned t1 = 39410U * blk[1 * step] - 58980U * blk[7 * step]; \
+    const unsigned t2 =-33410U * blk[5 * step] -167963U * blk[3 * step]; \
+    const unsigned t3 = 33410U * blk[3 * step] -167963U * blk[5 * step]; \
+    const unsigned t4 =          blk[3 * step] +          blk[7 * step]; \
+    const unsigned t5 =          blk[1 * step] +          blk[5 * step]; \
+    const unsigned t6 = 77062U * t4            + 51491U * t5;            \
+    const unsigned t7 = 77062U * t5            - 51491U * t4;            \
+    const unsigned t8 = 35470U * blk[2 * step] - 85623U * blk[6 * step]; \
+    const unsigned t9 = 35470U * blk[6 * step] + 85623U * blk[2 * step]; \
+    const unsigned tA = SOP(blk[0 * step] - blk[4 * step]);              \
+    const unsigned tB = SOP(blk[0 * step] + blk[4 * step]);              \
                                                                     \
-    blk[0 * step] = (  t1 + t6  + t9 + tB) >> shift;                \
-    blk[1 * step] = (  t3 + t7  + t8 + tA) >> shift;                \
-    blk[2 * step] = (  t2 + t6  - t8 + tA) >> shift;                \
-    blk[3 * step] = (  t0 + t7  - t9 + tB) >> shift;                \
-    blk[4 * step] = (-(t0 + t7) - t9 + tB) >> shift;                \
-    blk[5 * step] = (-(t2 + t6) - t8 + tA) >> shift;                \
-    blk[6 * step] = (-(t3 + t7) + t8 + tA) >> shift;                \
-    blk[7 * step] = (-(t1 + t6) + t9 + tB) >> shift;                \
+    blk[0 * step] = (int)(  t1 + t6  + t9 + tB) >> shift;                \
+    blk[1 * step] = (int)(  t3 + t7  + t8 + tA) >> shift;                \
+    blk[2 * step] = (int)(  t2 + t6  - t8 + tA) >> shift;                \
+    blk[3 * step] = (int)(  t0 + t7  - t9 + tB) >> shift;                \
+    blk[4 * step] = (int)(-(t0 + t7) - t9 + tB) >> shift;                \
+    blk[5 * step] = (int)(-(t2 + t6) - t8 + tA) >> shift;                \
+    blk[6 * step] = (int)(-(t3 + t7) + t8 + tA) >> shift;                \
+    blk[7 * step] = (int)(-(t1 + t6) + t9 + tB) >> shift;                \
 
-#define SOP_ROW(a) (((a) << 16) + 0x2000)
-#define SOP_COL(a) (((a) + 32) << 16)
+#define SOP_ROW(a) (((a) * (1U << 16)) + 0x2000)
+#define SOP_COL(a) (((a) + 32) * (1U << 16))
 
 void ff_mss34_dct_put(uint8_t *dst, ptrdiff_t stride, int *block)
 {