[FFmpeg-cvslog] vp3dec: Check coefficient index in vp3_dequant()

[Reinhard Tartler]

ffmpeg | branch: release/0.8 | Reinhard Tartler <siretart at tauware.de> | Sun Dec  4 10:10:33 2011 +0100| [bba709214a51ffd665a67404d3beb3727bb3f319] | committer: Reinhard Tartler

vp3dec: Check coefficient index in vp3_dequant()

Based on a patch by Michael Niedermayer <michaelni at gmx.at>

Fixes NGS00145, CVE-2011-4352

Found-by: Phillip Langlois
Signed-off-by: Reinhard Tartler <siretart at tauware.de>

(cherry picked from commit 8b94df0f2047e9728cb872adc9e64557b7a5152f)

Signed-off-by: Reinhard Tartler <siretart at tauware.de>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=bba709214a51ffd665a67404d3beb3727bb3f319
---

 libavcodec/vp3.c |   14 ++++++++++++--
 1 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/libavcodec/vp3.c b/libavcodec/vp3.c
index c117a64..890db5c 100644
--- a/libavcodec/vp3.c
+++ b/libavcodec/vp3.c
@@ -1291,6 +1291,10 @@ static inline int vp3_dequant(Vp3DecodeContext *s, Vp3Fragment *frag,
         case 1: // zero run
             s->dct_tokens[plane][i]++;
             i += (token >> 2) & 0x7f;
+            if (i > 63) {
+                av_log(s->avctx, AV_LOG_ERROR, "Coefficient index overflow\n");
+                return i;
+            }
             block[perm[i]] = (token >> 9) * dequantizer[perm[i]];
             i++;
             break;
@@ -1493,7 +1497,10 @@ static void render_slice(Vp3DecodeContext *s, int slice)
                     /* invert DCT and place (or add) in final output */
 
                     if (s->all_fragments[i].coding_method == MODE_INTRA) {
-                        vp3_dequant(s, s->all_fragments + i, plane, 0, block);
+                        int index;
+                        index = vp3_dequant(s, s->all_fragments + i, plane, 0, block);
+                        if (index > 63)
+                            continue;
                         if(s->avctx->idct_algo!=FF_IDCT_VP3)
                             block[0] += 128<<3;
                         s->dsp.idct_put(
@@ -1501,7 +1508,10 @@ static void render_slice(Vp3DecodeContext *s, int slice)
                             stride,
                             block);
                     } else {
-                        if (vp3_dequant(s, s->all_fragments + i, plane, 1, block)) {
+                        int index = vp3_dequant(s, s->all_fragments + i, plane, 1, block);
+                        if (index > 63)
+                            continue;
+                        if (index > 0) {
                         s->dsp.idct_add(
                             output_plane + first_pixel,
                             stride,