[FFmpeg-cvslog] avcodec/txd: Move input size checks before allocation

[Michael Niedermayer]

ffmpeg | branch: master | Michael Niedermayer <michael at niedermayer.cc> | Sun Nov 14 18:09:17 2021 +0100| [ff23cee200239a8368515c5d8ba94fc769c3fea2] | committer: Michael Niedermayer

avcodec/txd: Move input size checks before allocation

Fixes: Timeout
Fixes: 40882/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_TXD_fuzzer-4893364584054784

Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=ff23cee200239a8368515c5d8ba94fc769c3fea2
---

 libavcodec/txd.c | 30 +++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/libavcodec/txd.c b/libavcodec/txd.c
index 86d60f3377..93e4fa9605 100644
--- a/libavcodec/txd.c
+++ b/libavcodec/txd.c
@@ -65,8 +65,26 @@ static int txd_decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
 
     if (depth == 8) {
         avctx->pix_fmt = AV_PIX_FMT_PAL8;
-    } else if (depth == 16 || depth == 32) {
+        if (bytestream2_get_bytes_left(&gb) < w * h + 4 * 256)
+            return AVERROR_INVALIDDATA;
+    } else if (depth == 16) {
         avctx->pix_fmt = AV_PIX_FMT_RGBA;
+        switch (d3d_format) {
+        case 0:
+            if (!(flags & 1))
+                goto unsupported;
+        case TXD_DXT1:
+            if (bytestream2_get_bytes_left(&gb) < AV_CEIL_RSHIFT(w, 2) * AV_CEIL_RSHIFT(h, 2) * 8 + 4)
+                return AVERROR_INVALIDDATA;
+            break;
+        case TXD_DXT3:
+            if (bytestream2_get_bytes_left(&gb) < AV_CEIL_RSHIFT(w, 2) * AV_CEIL_RSHIFT(h, 2) * 16 + 4)
+                return AVERROR_INVALIDDATA;
+        }
+    } else if (depth == 32) {
+        avctx->pix_fmt = AV_PIX_FMT_RGBA;
+        if (bytestream2_get_bytes_left(&gb) < h * w * 4)
+            return AVERROR_INVALIDDATA;
     } else {
         avpriv_report_missing_feature(avctx, "Color depth of %u", depth);
         return AVERROR_PATCHWELCOME;
@@ -92,8 +110,6 @@ static int txd_decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
             v = bytestream2_get_be32(&gb);
             pal[y] = (v >> 8) + (v << 24);
         }
-        if (bytestream2_get_bytes_left(&gb) < w * h)
-            return AVERROR_INVALIDDATA;
         bytestream2_skip(&gb, 4);
         for (y=0; y<h; y++) {
             bytestream2_get_buffer(&gb, ptr, w);
@@ -103,11 +119,7 @@ static int txd_decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
         bytestream2_skip(&gb, 4);
         switch (d3d_format) {
         case 0:
-            if (!(flags & 1))
-                goto unsupported;
         case TXD_DXT1:
-            if (bytestream2_get_bytes_left(&gb) < AV_CEIL_RSHIFT(w, 2) * AV_CEIL_RSHIFT(h, 2) * 8)
-                return AVERROR_INVALIDDATA;
             for (j = 0; j < avctx->height; j += 4) {
                 for (i = 0; i < avctx->width; i += 4) {
                     uint8_t *p = ptr + i * 4 + j * stride;
@@ -117,8 +129,6 @@ static int txd_decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
             }
             break;
         case TXD_DXT3:
-            if (bytestream2_get_bytes_left(&gb) < AV_CEIL_RSHIFT(w, 2) * AV_CEIL_RSHIFT(h, 2) * 16)
-                return AVERROR_INVALIDDATA;
             for (j = 0; j < avctx->height; j += 4) {
                 for (i = 0; i < avctx->width; i += 4) {
                     uint8_t *p = ptr + i * 4 + j * stride;
@@ -134,8 +144,6 @@ static int txd_decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
         switch (d3d_format) {
         case 0x15:
         case 0x16:
-            if (bytestream2_get_bytes_left(&gb) < h * w * 4)
-                return AVERROR_INVALIDDATA;
             for (y=0; y<h; y++) {
                 bytestream2_get_buffer(&gb, ptr, w * 4);
                 ptr += stride;