[FFmpeg-cvslog] avcodec/dstdec: Check for overflow in build_filter()

[Michael Niedermayer]

ffmpeg | branch: release/3.4 | Michael Niedermayer <michael at niedermayer.cc> | Sat Sep 10 23:49:28 2022 +0200| [cdeb121c6a1fae5d785cdba7941046e3f17fc470] | committer: Michael Niedermayer

avcodec/dstdec: Check for overflow in build_filter()

Fixes: signed integer overflow: 1917019860 + 265558963 cannot be represented in type 'int'
Fixes: 48798/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_DST_fuzzer-4833165046317056

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
(cherry picked from commit 8008940da5aa43895fd4574114309c3324249eab)
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=cdeb121c6a1fae5d785cdba7941046e3f17fc470
---

 libavcodec/dstdec.c | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/libavcodec/dstdec.c b/libavcodec/dstdec.c
index d7a82f34ec..7a51f6772a 100644
--- a/libavcodec/dstdec.c
+++ b/libavcodec/dstdec.c
@@ -214,7 +214,7 @@ static uint8_t prob_dst_x_bit(int c)
     return (ff_reverse[c & 127] >> 1) + 1;
 }
 
-static void build_filter(int16_t table[DST_MAX_ELEMENTS][16][256], const Table *fsets)
+static int build_filter(int16_t table[DST_MAX_ELEMENTS][16][256], const Table *fsets)
 {
     int i, j, k, l;
 
@@ -225,14 +225,17 @@ static void build_filter(int16_t table[DST_MAX_ELEMENTS][16][256], const Table *
             int total = av_clip(length - j * 8, 0, 8);
 
             for (k = 0; k < 256; k++) {
-                int v = 0;
+                int64_t v = 0;
 
                 for (l = 0; l < total; l++)
                     v += (((k >> l) & 1) * 2 - 1) * fsets->coeff[i][j * 8 + l];
+                if ((int16_t)v != v)
+                    return AVERROR_INVALIDDATA;
                 table[i][j][k] = v;
             }
         }
     }
+    return 0;
 }
 
 static int decode_frame(AVCodecContext *avctx, void *data,
@@ -328,7 +331,9 @@ static int decode_frame(AVCodecContext *avctx, void *data,
         return AVERROR_INVALIDDATA;
     ac_init(ac, gb);
 
-    build_filter(s->filter, &s->fsets);
+    ret = build_filter(s->filter, &s->fsets);
+    if (ret < 0)
+        return ret;
 
     memset(s->status, 0xAA, sizeof(s->status));
     memset(dsd, 0, frame->nb_samples * 4 * avctx->channels);