[FFmpeg-cvslog] avcodec/hevc_cabac: Fix multiple integer overflows

[Michael Niedermayer]

ffmpeg | branch: release/2.7 | Michael Niedermayer <michael at niedermayer.cc> | Fri Nov 27 22:45:46 2015 +0100| [694416e327c2fb6495cbc62f63acec65d8c5446c] | committer: Michael Niedermayer

avcodec/hevc_cabac: Fix multiple integer overflows

Fixes: 04ec80eefa77aecd7a49a442cc02baea/asan_heap-oob_19544fa_3303_1905796cd9d8e15f86d664332caabc00.bit

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
(cherry picked from commit d5028f61e44b7607b6a547f218f7d85217490a5b)

Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=694416e327c2fb6495cbc62f63acec65d8c5446c
---

 libavcodec/hevc_cabac.c |   10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/libavcodec/hevc_cabac.c b/libavcodec/hevc_cabac.c
index 3d16896..c1c9f20 100644
--- a/libavcodec/hevc_cabac.c
+++ b/libavcodec/hevc_cabac.c
@@ -883,11 +883,13 @@ static av_always_inline int mvd_decode(HEVCContext *s)
     int k = 1;
 
     while (k < CABAC_MAX_BIN && get_cabac_bypass(&s->HEVClc->cc)) {
-        ret += 1 << k;
+        ret += 1U << k;
         k++;
     }
-    if (k == CABAC_MAX_BIN)
+    if (k == CABAC_MAX_BIN) {
         av_log(s->avctx, AV_LOG_ERROR, "CABAC_MAX_BIN : %d\n", k);
+        return 0;
+    }
     while (k--)
         ret += get_cabac_bypass(&s->HEVClc->cc) << k;
     return get_cabac_bypass_sign(&s->HEVClc->cc, -ret);
@@ -1025,8 +1027,10 @@ static av_always_inline int coeff_abs_level_remaining_decode(HEVCContext *s, int
 
     while (prefix < CABAC_MAX_BIN && get_cabac_bypass(&s->HEVClc->cc))
         prefix++;
-    if (prefix == CABAC_MAX_BIN)
+    if (prefix == CABAC_MAX_BIN) {
         av_log(s->avctx, AV_LOG_ERROR, "CABAC_MAX_BIN : %d\n", prefix);
+        return 0;
+    }
     if (prefix < 3) {
         for (i = 0; i < rc_rice_param; i++)
             suffix = (suffix << 1) | get_cabac_bypass(&s->HEVClc->cc);