[FFmpeg-cvslog] vorbis: Check the vlc value in setup_classifs

[Luca Barbato]

ffmpeg | branch: master | Luca Barbato <lu_zero at gentoo.org> | Tue Mar  3 11:05:15 2015 +0100| [0025f7408a0fab2cab4a950064e4784a67463994] | committer: Luca Barbato

vorbis: Check the vlc value in setup_classifs

The valid returned values are always at most 11bit.
Remove the previous check that assumed larger values plausible and
use a signed integer to check get_vlc2 return values.

CC: libav-stable at libav.org

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=0025f7408a0fab2cab4a950064e4784a67463994
---

 libavcodec/vorbisdec.c |   30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/libavcodec/vorbisdec.c b/libavcodec/vorbisdec.c
index e156c08..0865dc8 100644
--- a/libavcodec/vorbisdec.c
+++ b/libavcodec/vorbisdec.c
@@ -1311,7 +1311,7 @@ static av_always_inline int setup_classifs(vorbis_context *vc,
     int p, j, i;
     unsigned c_p_c         = codebook->dimensions;
     unsigned inverse_class = ff_inverse[vr->classifications];
-    unsigned temp, temp2;
+    int temp, temp2;
     for (p = 0, j = 0; j < ch_used; ++j) {
         if (!do_not_decode[j]) {
             temp = get_vlc2(&vc->gb, codebook->vlc.table,
@@ -1319,22 +1319,18 @@ static av_always_inline int setup_classifs(vorbis_context *vc,
 
             av_dlog(NULL, "Classword: %u\n", temp);
 
-            if (temp <= 65536) {
-                for (i = partition_count + c_p_c - 1; i >= partition_count; i--) {
-                    temp2 = (((uint64_t)temp) * inverse_class) >> 32;
+            if (temp < 0) {
+                av_log(vc->avctx, AV_LOG_ERROR,
+                       "Invalid vlc code decoding %d channel.", j);
+                return AVERROR_INVALIDDATA;
+            }
 
-                    if (i < vr->ptns_to_read)
-                        vr->classifs[p + i] = temp - temp2 * vr->classifications;
-                    temp = temp2;
-                }
-            } else {
-                for (i = partition_count + c_p_c - 1; i >= partition_count; i--) {
-                    temp2 = temp / vr->classifications;
+            for (i = partition_count + c_p_c - 1; i >= partition_count; i--) {
+                temp2 = (((uint64_t)temp) * inverse_class) >> 32;
 
-                    if (i < vr->ptns_to_read)
-                        vr->classifs[p + i] = temp - temp2 * vr->classifications;
-                    temp = temp2;
-                }
+                if (i < vr->ptns_to_read)
+                    vr->classifs[p + i] = temp - temp2 * vr->classifications;
+                temp = temp2;
             }
         }
         p += vr->ptns_to_read;
@@ -1384,7 +1380,9 @@ static av_always_inline int vorbis_residue_decode_internal(vorbis_context *vc,
         voffset = vr->begin;
         for (partition_count = 0; partition_count < ptns_to_read;) {  // SPEC        error
             if (!pass) {
-                setup_classifs(vc, vr, do_not_decode, ch_used, partition_count);
+                int ret = setup_classifs(vc, vr, do_not_decode, ch_used, partition_count);
+                if (ret < 0)
+                    return ret;
             }
             for (i = 0; (i < c_p_c) && (partition_count < ptns_to_read); ++i) {
                 for (j_times_ptns_to_read = 0, j = 0; j < ch_used; ++j) {