[FFmpeg-cvslog] mpegvideo: Make the table reallocation more robust.

[Michael Niedermayer]

ffmpeg | branch: master | Michael Niedermayer <michaelni at gmx.at> | Thu Mar 28 23:46:13 2013 +0100| [6ae03353de664d670b32c33726f81758646f99e6] | committer: Michael Niedermayer

mpegvideo: Make the table reallocation more robust.

This fixes out of array writes after resolution changes
No FFmpeg releases are known to be affected by this

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni at gmx.at>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=6ae03353de664d670b32c33726f81758646f99e6
---

 libavcodec/mpegvideo.c |   13 +++++++++++--
 libavcodec/mpegvideo.h |    3 +++
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c
index f4739c4..2969a8c 100644
--- a/libavcodec/mpegvideo.c
+++ b/libavcodec/mpegvideo.c
@@ -294,6 +294,9 @@ static void free_picture_tables(Picture *pic)
 {
     int i;
 
+    pic->alloc_mb_width  =
+    pic->alloc_mb_height = 0;
+
     av_buffer_unref(&pic->mb_var_buf);
     av_buffer_unref(&pic->mc_mb_var_buf);
     av_buffer_unref(&pic->mb_mean_buf);
@@ -343,6 +346,9 @@ static int alloc_picture_tables(MpegEncContext *s, Picture *pic)
         }
     }
 
+    pic->alloc_mb_width  = s->mb_width;
+    pic->alloc_mb_height = s->mb_height;
+
     return 0;
 }
 
@@ -380,8 +386,8 @@ int ff_alloc_picture(MpegEncContext *s, Picture *pic, int shared)
     int i, ret;
 
     if (pic->qscale_table_buf)
-        if (pic->mbskip_table_buf->size < s->mb_stride * s->mb_height + 2 ||
-            pic->qscale_table_buf->size < s->mb_stride * (s->mb_height + 1) + 1 + s->mb_stride)
+        if (   pic->alloc_mb_width  != s->mb_width
+            || pic->alloc_mb_height != s->mb_height)
             free_picture_tables(pic);
 
     if (shared) {
@@ -490,6 +496,9 @@ do {\
         dst->ref_index[i]  = src->ref_index[i];
     }
 
+    dst->alloc_mb_width  = src->alloc_mb_width;
+    dst->alloc_mb_height = src->alloc_mb_height;
+
     return 0;
 }
 
diff --git a/libavcodec/mpegvideo.h b/libavcodec/mpegvideo.h
index addbde0..e62700c 100644
--- a/libavcodec/mpegvideo.h
+++ b/libavcodec/mpegvideo.h
@@ -119,6 +119,9 @@ typedef struct Picture{
     AVBufferRef *mc_mb_var_buf;
     uint16_t *mc_mb_var;        ///< Table for motion compensated MB variances
 
+    int alloc_mb_width;         ///< mb_width used to allocate tables
+    int alloc_mb_height;        ///< mb_height used to allocate tables
+
     AVBufferRef *mb_mean_buf;
     uint8_t *mb_mean;           ///< Table for MB luminance