[FFmpeg-cvslog] Fix crashes in vorbis decoding found by zzuf

[Jason Garrett-Glaser]

ffmpeg | branch: release/0.5 | Jason Garrett-Glaser <darkshikari at gmail.com> | Sun Feb 13 20:41:13 2011 +0100| [d6860fb653ed42a9d35e134f843f03cc049b74f1] | committer: Reinhard Tartler

Fix crashes in vorbis decoding found by zzuf
Fixes issue 2322.

Originally committed as revision 25591 to svn://svn.ffmpeg.org/ffmpeg/trunk
(cherry picked from commit 3dde66752d59dfdd0f3727efd66e7202b3c75078)

Addresses: CVE-2010-4704

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=d6860fb653ed42a9d35e134f843f03cc049b74f1
---

 libavcodec/vorbis_dec.c |   28 +++++++++++++++++++---------
 1 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/libavcodec/vorbis_dec.c b/libavcodec/vorbis_dec.c
index 9875657..541a406 100644
--- a/libavcodec/vorbis_dec.c
+++ b/libavcodec/vorbis_dec.c
@@ -60,8 +60,8 @@ typedef struct vorbis_floor0_s vorbis_floor0;
 typedef struct vorbis_floor1_s vorbis_floor1;
 struct vorbis_context_s;
 typedef
-uint_fast8_t (* vorbis_floor_decode_func)
-             (struct vorbis_context_s *, vorbis_floor_data *, float *);
+int (* vorbis_floor_decode_func)
+    (struct vorbis_context_s *, vorbis_floor_data *, float *);
 typedef struct {
     uint_fast8_t floor_type;
     vorbis_floor_decode_func decode;
@@ -443,14 +443,14 @@ static int vorbis_parse_setup_hdr_tdtransforms(vorbis_context *vc) {
 
 // Process floors part
 
-static uint_fast8_t vorbis_floor0_decode(vorbis_context *vc,
+static int vorbis_floor0_decode(vorbis_context *vc,
                                          vorbis_floor_data *vfu, float *vec);
 static void create_map( vorbis_context * vc, uint_fast8_t floor_number );
-static uint_fast8_t vorbis_floor1_decode(vorbis_context *vc,
+static int vorbis_floor1_decode(vorbis_context *vc,
                                          vorbis_floor_data *vfu, float *vec);
 static int vorbis_parse_setup_hdr_floors(vorbis_context *vc) {
     GetBitContext *gb=&vc->gb;
-    uint_fast16_t i,j,k;
+    int i,j,k;
 
     vc->floor_count=get_bits(gb, 6)+1;
 
@@ -1038,7 +1038,7 @@ static av_cold int vorbis_decode_init(AVCodecContext *avccontext) {
 
 // Read and decode floor
 
-static uint_fast8_t vorbis_floor0_decode(vorbis_context *vc,
+static int vorbis_floor0_decode(vorbis_context *vc,
                                          vorbis_floor_data *vfu, float *vec) {
     vorbis_floor0 * vf=&vfu->t0;
     float * lsp=vf->lsp;
@@ -1062,6 +1062,9 @@ static uint_fast8_t vorbis_floor0_decode(vorbis_context *vc,
         }
         AV_DEBUG( "floor0 dec: booknumber: %u\n", book_idx );
         codebook=vc->codebooks[vf->book_list[book_idx]];
+        /* Invalid codebook! */
+        if (!codebook.codevectors)
+            return -1;
 
         while (lsp_len<vf->order) {
             int vec_off;
@@ -1151,7 +1154,7 @@ static uint_fast8_t vorbis_floor0_decode(vorbis_context *vc,
     return 0;
 }
 
-static uint_fast8_t vorbis_floor1_decode(vorbis_context *vc, vorbis_floor_data *vfu, float *vec) {
+static int vorbis_floor1_decode(vorbis_context *vc, vorbis_floor_data *vfu, float *vec) {
     vorbis_floor1 * vf=&vfu->t1;
     GetBitContext *gb=&vc->gb;
     uint_fast16_t range_v[4]={ 256, 128, 86, 64 };
@@ -1527,14 +1530,21 @@ static int vorbis_parse_audio_packet(vorbis_context *vc) {
 
     for(i=0;i<vc->audio_channels;++i) {
         vorbis_floor *floor;
+        int ret;
         if (mapping->submaps>1) {
             floor=&vc->floors[mapping->submap_floor[mapping->mux[i]]];
         } else {
             floor=&vc->floors[mapping->submap_floor[0]];
         }
 
-        no_residue[i]=floor->decode(vc, &floor->data, ch_floor_ptr);
-        ch_floor_ptr+=blocksize/2;
+        ret = floor->decode(vc, &floor->data, ch_floor_ptr);
+
+        if (ret < 0) {
+            av_log(vc->avccontext, AV_LOG_ERROR, "Invalid codebook in vorbis_floor_decode.\n");
+            return -1;
+        }
+        no_residue[i] = ret;
+        ch_floor_ptr += blocksize / 2;
     }
 
 // Nonzero vector propagate