[FFmpeg-cvslog] vorbis: Validate that the floor 1 X values contain no duplicates.

[Alex Converse]

ffmpeg | branch: release/0.7 | Alex Converse <alex.converse at gmail.com> | Mon Jun  4 18:27:03 2012 -0700| [d6e250abfc36b239ef0c1fc9d45d588b853bfcb9] | committer: Anton Khirnov

vorbis: Validate that the floor 1 X values contain no duplicates.

Duplicate values in this vector are explicitly banned by the Vorbis I spec
and cause divide-by-zero crashes later on.
(cherry picked from commit ecf79c4d3e8baaf2f303278ef81db6f8407656bc)

Signed-off-by: Reinhard Tartler <siretart at tauware.de>
(cherry picked from commit 9aaaeba45c41cf2b3fa4100abbdee7437428f93c)

Signed-off-by: Anton Khirnov <anton at khirnov.net>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=d6e250abfc36b239ef0c1fc9d45d588b853bfcb9
---

 libavcodec/vorbis.c    |    9 ++++++++-
 libavcodec/vorbis.h    |    3 ++-
 libavcodec/vorbisdec.c |    6 +++++-
 libavcodec/vorbisenc.c |    3 ++-
 4 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/libavcodec/vorbis.c b/libavcodec/vorbis.c
index 20be707..e0eda46 100644
--- a/libavcodec/vorbis.c
+++ b/libavcodec/vorbis.c
@@ -117,7 +117,8 @@ int ff_vorbis_len2vlc(uint8_t *bits, uint32_t *codes, unsigned num)
     return 0;
 }
 
-void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values)
+int ff_vorbis_ready_floor1_list(AVCodecContext *avccontext,
+                                vorbis_floor1_entry *list, int values)
 {
     int i;
     list[0].sort = 0;
@@ -141,6 +142,11 @@ void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values)
     for (i = 0; i < values - 1; i++) {
         int j;
         for (j = i + 1; j < values; j++) {
+            if (list[i].x == list[j].x) {
+                av_log(avccontext, AV_LOG_ERROR,
+                       "Duplicate value found in floor 1 X coordinates\n");
+                return AVERROR_INVALIDDATA;
+            }
             if (list[list[i].sort].x > list[list[j].sort].x) {
                 int tmp = list[i].sort;
                 list[i].sort = list[j].sort;
@@ -148,6 +154,7 @@ void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values)
             }
         }
     }
+    return 0;
 }
 
 static inline void render_line_unrolled(intptr_t x, int y, int x1,
diff --git a/libavcodec/vorbis.h b/libavcodec/vorbis.h
index 8501e0a..b4346fb 100644
--- a/libavcodec/vorbis.h
+++ b/libavcodec/vorbis.h
@@ -36,7 +36,8 @@ typedef struct {
     uint16_t high;
 } vorbis_floor1_entry;
 
-void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values);
+int ff_vorbis_ready_floor1_list(AVCodecContext *avccontext,
+                                vorbis_floor1_entry *list, int values);
 unsigned int ff_vorbis_nth_root(unsigned int x, unsigned int n); // x^(1/n)
 int ff_vorbis_len2vlc(uint8_t *bits, uint32_t *codes, unsigned num);
 void ff_vorbis_floor1_render_list(vorbis_floor1_entry * list, int values,
diff --git a/libavcodec/vorbisdec.c b/libavcodec/vorbisdec.c
index d6850b7..3805d27 100644
--- a/libavcodec/vorbisdec.c
+++ b/libavcodec/vorbisdec.c
@@ -559,7 +559,11 @@ static int vorbis_parse_setup_hdr_floors(vorbis_context *vc)
             }
 
 // Precalculate order of x coordinates - needed for decode
-            ff_vorbis_ready_floor1_list(floor_setup->data.t1.list, floor_setup->data.t1.x_list_dim);
+            if (ff_vorbis_ready_floor1_list(vc->avccontext,
+                                            floor_setup->data.t1.list,
+                                            floor_setup->data.t1.x_list_dim)) {
+                return AVERROR_INVALIDDATA;
+            }
         } else if (floor_setup->floor_type == 0) {
             unsigned max_codebook_dim = 0;
 
diff --git a/libavcodec/vorbisenc.c b/libavcodec/vorbisenc.c
index ad69c61..7311c05 100644
--- a/libavcodec/vorbisenc.c
+++ b/libavcodec/vorbisenc.c
@@ -336,7 +336,8 @@ static int create_vorbis_context(vorbis_enc_context *venc,
         };
         fc->list[i].x = a[i - 2];
     }
-    ff_vorbis_ready_floor1_list(fc->list, fc->values);
+    if (ff_vorbis_ready_floor1_list(avccontext, fc->list, fc->values))
+        return AVERROR(EINVAL);
 
     venc->nresidues = 1;
     venc->residues  = av_malloc(sizeof(vorbis_enc_residue) * venc->nresidues);