[FFmpeg-cvslog] avcodec/hapdec: Check tex_size more strictly and before using it

[Michael Niedermayer]

ffmpeg | branch: master | Michael Niedermayer <michael at niedermayer.cc> | Sat Mar 28 20:36:00 2020 +0100| [81fe316ad9852a3dfe46b4dc919ed1709b217671] | committer: Michael Niedermayer

avcodec/hapdec: Check tex_size more strictly and before using it

Fixes: OOM
Fixes: 20774/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_HAP_fuzzer-5678608951803904
Fixes: 20956/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_HAP_fuzzer-5713643025203200

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=81fe316ad9852a3dfe46b4dc919ed1709b217671
---

 libavcodec/hapdec.c | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/libavcodec/hapdec.c b/libavcodec/hapdec.c
index d23ceb5cef..ab364aa790 100644
--- a/libavcodec/hapdec.c
+++ b/libavcodec/hapdec.c
@@ -305,7 +305,6 @@ static int hap_decode(AVCodecContext *avctx, void *data,
     HapContext *ctx = avctx->priv_data;
     ThreadFrame tframe;
     int ret, i, t;
-    int tex_size;
     int section_size;
     enum HapSectionType section_type;
     int start_texture_section = 0;
@@ -342,6 +341,13 @@ static int hap_decode(AVCodecContext *avctx, void *data,
         if (ret < 0)
             return ret;
 
+        if (ctx->tex_size != (avctx->coded_width  / TEXTURE_BLOCK_W)
+            *(avctx->coded_height / TEXTURE_BLOCK_H)
+            *tex_rat[t]) {
+            av_log(avctx, AV_LOG_ERROR, "uncompressed size mismatches\n");
+            return AVERROR_INVALIDDATA;
+        }
+
         start_texture_section += ctx->texture_section_size + 4;
 
         if (avctx->codec->update_thread_context)
@@ -349,9 +355,16 @@ static int hap_decode(AVCodecContext *avctx, void *data,
 
         /* Unpack the DXT texture */
         if (hap_can_use_tex_in_place(ctx)) {
+            int tex_size;
             /* Only DXTC texture compression in a contiguous block */
             ctx->tex_data = ctx->gbc.buffer;
             tex_size = FFMIN(ctx->texture_section_size, bytestream2_get_bytes_left(&ctx->gbc));
+            if (tex_size < (avctx->coded_width  / TEXTURE_BLOCK_W)
+                *(avctx->coded_height / TEXTURE_BLOCK_H)
+                *tex_rat[t]) {
+                av_log(avctx, AV_LOG_ERROR, "Insufficient data\n");
+                return AVERROR_INVALIDDATA;
+            }
         } else {
             /* Perform the second-stage decompression */
             ret = av_reallocp(&ctx->tex_buf, ctx->tex_size);
@@ -367,14 +380,6 @@ static int hap_decode(AVCodecContext *avctx, void *data,
             }
 
             ctx->tex_data = ctx->tex_buf;
-            tex_size = ctx->tex_size;
-        }
-
-        if (tex_size < (avctx->coded_width  / TEXTURE_BLOCK_W)
-            *(avctx->coded_height / TEXTURE_BLOCK_H)
-            *tex_rat[t]) {
-            av_log(avctx, AV_LOG_ERROR, "Insufficient data\n");
-            return AVERROR_INVALIDDATA;
         }
 
         /* Use the decompress function on the texture, one block per thread */