[FFmpeg-cvslog] avcodec/proresdec2: Check bits in DECODE_CODEWORD(), fixes invalid shift

[Michael Niedermayer]

ffmpeg | branch: release/3.3 | Michael Niedermayer <michael at niedermayer.cc> | Fri Sep 22 20:45:28 2017 +0200| [4bc16930efdb56dc53caf615cec7085435bf10e0] | committer: Michael Niedermayer

avcodec/proresdec2: Check bits in DECODE_CODEWORD(), fixes invalid shift

Fixes: runtime error: shift exponent 42 is too large for 32-bit type 'unsigned int'
Fixes: 3410/clusterfuzz-testcase-minimized-5313377960198144

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
(cherry picked from commit 4f5eaf0b5956e492ee5023929669b1d09aaf6299)
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=4bc16930efdb56dc53caf615cec7085435bf10e0
---

 libavcodec/proresdec2.c | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/libavcodec/proresdec2.c b/libavcodec/proresdec2.c
index ff46bcfde0..779ae6a999 100644
--- a/libavcodec/proresdec2.c
+++ b/libavcodec/proresdec2.c
@@ -267,6 +267,8 @@ static int decode_picture_header(AVCodecContext *avctx, const uint8_t *buf, cons
                                                                         \
         if (q > switch_bits) { /* exp golomb */                         \
             bits = exp_order - switch_bits + (q<<1);                    \
+            if (bits > MIN_CACHE_BITS)                                  \
+                return AVERROR_INVALIDDATA;                             \
             val = SHOW_UBITS(re, gb, bits) - (1 << exp_order) +         \
                 ((switch_bits + 1) << rice_order);                      \
             SKIP_BITS(re, gb, bits);                                    \
@@ -286,7 +288,7 @@ static int decode_picture_header(AVCodecContext *avctx, const uint8_t *buf, cons
 
 static const uint8_t dc_codebook[7] = { 0x04, 0x28, 0x28, 0x4D, 0x4D, 0x70, 0x70};
 
-static av_always_inline void decode_dc_coeffs(GetBitContext *gb, int16_t *out,
+static av_always_inline int decode_dc_coeffs(GetBitContext *gb, int16_t *out,
                                               int blocks_per_slice)
 {
     int16_t prev_dc;
@@ -310,6 +312,7 @@ static av_always_inline void decode_dc_coeffs(GetBitContext *gb, int16_t *out,
         out[0] = prev_dc;
     }
     CLOSE_READER(re, gb);
+    return 0;
 }
 
 // adaptive codebook switching lut according to previous run/level values
@@ -376,7 +379,8 @@ static int decode_slice_luma(AVCodecContext *avctx, SliceContext *slice,
 
     init_get_bits(&gb, buf, buf_size << 3);
 
-    decode_dc_coeffs(&gb, blocks, blocks_per_slice);
+    if ((ret = decode_dc_coeffs(&gb, blocks, blocks_per_slice)) < 0)
+        return ret;
     if ((ret = decode_ac_coeffs(avctx, &gb, blocks, blocks_per_slice)) < 0)
         return ret;
 
@@ -409,7 +413,8 @@ static int decode_slice_chroma(AVCodecContext *avctx, SliceContext *slice,
 
     init_get_bits(&gb, buf, buf_size << 3);
 
-    decode_dc_coeffs(&gb, blocks, blocks_per_slice);
+    if ((ret = decode_dc_coeffs(&gb, blocks, blocks_per_slice)) < 0)
+        return ret;
     if ((ret = decode_ac_coeffs(avctx, &gb, blocks, blocks_per_slice)) < 0)
         return ret;