[FFmpeg-cvslog] avcodec/rv10: check size of s->mb_width * s->mb_height

[Andreas Cadhalpun]

ffmpeg | branch: release/2.4 | Andreas Cadhalpun <Andreas.Cadhalpun at googlemail.com> | Mon Mar  2 20:27:26 2015 +0100| [d08db138e2109725ba2963f152b0a2b1fffded1b] | committer: Michael Niedermayer

avcodec/rv10: check size of s->mb_width * s->mb_height

If it doesn't fit into 12 bits it triggers an assertion.

Signed-off-by: Andreas Cadhalpun <Andreas.Cadhalpun at googlemail.com>
Signed-off-by: Michael Niedermayer <michaelni at gmx.at>
(cherry picked from commit 2578a546183da09d49d5bba8ab5e982dece1dede)

Signed-off-by: Michael Niedermayer <michaelni at gmx.at>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=d08db138e2109725ba2963f152b0a2b1fffded1b
---

 libavcodec/mpegvideo.h     |    2 +-
 libavcodec/mpegvideo_enc.c |    7 +++++--
 libavcodec/rv10enc.c       |    7 ++++++-
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/libavcodec/mpegvideo.h b/libavcodec/mpegvideo.h
index 87fe87f..63ce8e7 100644
--- a/libavcodec/mpegvideo.h
+++ b/libavcodec/mpegvideo.h
@@ -851,7 +851,7 @@ extern const uint8_t ff_aic_dc_scale_table[32];
 extern const uint8_t ff_h263_chroma_qscale_table[32];
 
 /* rv10.c */
-void ff_rv10_encode_picture_header(MpegEncContext *s, int picture_number);
+int ff_rv10_encode_picture_header(MpegEncContext *s, int picture_number);
 int ff_rv_decode_dc(MpegEncContext *s, int n);
 void ff_rv20_encode_picture_header(MpegEncContext *s, int picture_number);
 
diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c
index 61d7531..6b17855 100644
--- a/libavcodec/mpegvideo_enc.c
+++ b/libavcodec/mpegvideo_enc.c
@@ -3657,8 +3657,11 @@ static int encode_picture(MpegEncContext *s, int picture_number)
             ff_msmpeg4_encode_picture_header(s, picture_number);
         else if (CONFIG_MPEG4_ENCODER && s->h263_pred)
             ff_mpeg4_encode_picture_header(s, picture_number);
-        else if (CONFIG_RV10_ENCODER && s->codec_id == AV_CODEC_ID_RV10)
-            ff_rv10_encode_picture_header(s, picture_number);
+        else if (CONFIG_RV10_ENCODER && s->codec_id == AV_CODEC_ID_RV10) {
+            ret = ff_rv10_encode_picture_header(s, picture_number);
+            if (ret < 0)
+                return ret;
+        }
         else if (CONFIG_RV20_ENCODER && s->codec_id == AV_CODEC_ID_RV20)
             ff_rv20_encode_picture_header(s, picture_number);
         else if (CONFIG_FLV_ENCODER && s->codec_id == AV_CODEC_ID_FLV1)
diff --git a/libavcodec/rv10enc.c b/libavcodec/rv10enc.c
index 2541132..37efe6c 100644
--- a/libavcodec/rv10enc.c
+++ b/libavcodec/rv10enc.c
@@ -28,7 +28,7 @@
 #include "mpegvideo.h"
 #include "put_bits.h"
 
-void ff_rv10_encode_picture_header(MpegEncContext *s, int picture_number)
+int ff_rv10_encode_picture_header(MpegEncContext *s, int picture_number)
 {
     int full_frame= 0;
 
@@ -48,12 +48,17 @@ void ff_rv10_encode_picture_header(MpegEncContext *s, int picture_number)
     /* if multiple packets per frame are sent, the position at which
        to display the macroblocks is coded here */
     if(!full_frame){
+        if (s->mb_width * s->mb_height >= (1U << 12)) {
+            avpriv_report_missing_feature(s, "Encoding frames with 4096 or more macroblocks");
+            return AVERROR(ENOSYS);
+        }
         put_bits(&s->pb, 6, 0); /* mb_x */
         put_bits(&s->pb, 6, 0); /* mb_y */
         put_bits(&s->pb, 12, s->mb_width * s->mb_height);
     }
 
     put_bits(&s->pb, 3, 0);     /* ignored */
+    return 0;
 }
 
 FF_MPV_GENERIC_CLASS(rv10)