[FFmpeg-cvslog] avcodec/rv10: Execute whole size check earlier for rv20

[Michael Niedermayer]

ffmpeg | branch: release/4.4 | Michael Niedermayer <michael at niedermayer.cc> | Sat Apr 10 18:43:50 2021 +0200| [f4c5abe2d011f645d12226ec660bf70d94c73e4f] | committer: Michael Niedermayer

avcodec/rv10: Execute whole size check earlier for rv20

Fixes: Timeout
Fixes: 31380/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_RV20_fuzzer-5230899257016320

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
(cherry picked from commit f8556b632f4b29e54067ecba6f71ceb8af3806e4)
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=f4c5abe2d011f645d12226ec660bf70d94c73e4f
---

 libavcodec/rv10.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/libavcodec/rv10.c b/libavcodec/rv10.c
index bd70689cab..9c3a48c251 100644
--- a/libavcodec/rv10.c
+++ b/libavcodec/rv10.c
@@ -154,7 +154,7 @@ static int rv10_decode_picture_header(MpegEncContext *s)
     return mb_count;
 }
 
-static int rv20_decode_picture_header(RVDecContext *rv)
+static int rv20_decode_picture_header(RVDecContext *rv, int whole_size)
 {
     MpegEncContext *s = &rv->m;
     int seq, mb_pos, i, ret;
@@ -232,6 +232,10 @@ static int rv20_decode_picture_header(RVDecContext *rv)
                    "attempting to change resolution to %dx%d\n", new_w, new_h);
             if (av_image_check_size(new_w, new_h, 0, s->avctx) < 0)
                 return AVERROR_INVALIDDATA;
+
+            if (whole_size < (new_w + 15)/16 * ((new_h + 15)/16) / 8)
+                return AVERROR_INVALIDDATA;
+
             ff_mpv_common_end(s);
 
             // attempt to keep aspect during typical resolution switches
@@ -447,7 +451,7 @@ static int rv10_decode_packet(AVCodecContext *avctx, const uint8_t *buf,
     if (s->codec_id == AV_CODEC_ID_RV10)
         mb_count = rv10_decode_picture_header(s);
     else
-        mb_count = rv20_decode_picture_header(rv);
+        mb_count = rv20_decode_picture_header(rv, whole_size);
     if (mb_count < 0) {
         if (mb_count != ERROR_SKIP_FRAME)
             av_log(s->avctx, AV_LOG_ERROR, "HEADER ERROR\n");