[FFmpeg-cvslog] h264: stricter reference limit enforcement.

[Ronald S. Bultje]

ffmpeg | branch: master | Ronald S. Bultje <rsbultje at gmail.com> | Tue Mar 13 16:26:44 2012 -0700| [e0febda22d0e0fab094a9c886b0e0f0f662df1ef] | committer: Ronald S. Bultje

h264: stricter reference limit enforcement.

Progressive images can have only 16 references, error out if there are
more, since the data is almost certainly corrupt, and the invalid value
will lead to random crashes or invalid writes later on.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable at libav.org

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=e0febda22d0e0fab094a9c886b0e0f0f662df1ef
---

 libavcodec/h264.c |   13 ++++++++-----
 1 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index c3ef3f0..64f676c 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -3021,6 +3021,8 @@ static int decode_slice_header(H264Context *h, H264Context *h0){
     h->ref_count[1]= h->pps.ref_count[1];
 
     if(h->slice_type_nos != AV_PICTURE_TYPE_I){
+        int max_refs = s->picture_structure == PICT_FRAME ? 16 : 32;
+
         if(h->slice_type_nos == AV_PICTURE_TYPE_B){
             h->direct_spatial_mv_pred= get_bits1(&s->gb);
         }
@@ -3030,13 +3032,14 @@ static int decode_slice_header(H264Context *h, H264Context *h0){
             h->ref_count[0]= get_ue_golomb(&s->gb) + 1;
             if(h->slice_type_nos==AV_PICTURE_TYPE_B)
                 h->ref_count[1]= get_ue_golomb(&s->gb) + 1;
+        }
 
-            if(h->ref_count[0]-1 > 32-1 || h->ref_count[1]-1 > 32-1){
-                av_log(h->s.avctx, AV_LOG_ERROR, "reference overflow\n");
-                h->ref_count[0]= h->ref_count[1]= 1;
-                return -1;
-            }
+        if (h->ref_count[0] > max_refs || h->ref_count[1] > max_refs) {
+            av_log(h->s.avctx, AV_LOG_ERROR, "reference overflow\n");
+            h->ref_count[0] = h->ref_count[1] = 1;
+            return AVERROR_INVALIDDATA;
         }
+
         if(h->slice_type_nos == AV_PICTURE_TYPE_B)
             h->list_count= 2;
         else