[FFmpeg-cvslog] avcodec/utils: do "calc from frame_bytes, channels, and block_align" in 64bit

[Michael Niedermayer]

ffmpeg | branch: release/4.4 | Michael Niedermayer <michael at niedermayer.cc> | Wed Apr 28 16:50:13 2021 +0200| [a21c64199cddbea0356c98afa756fcf158634fd8] | committer: Michael Niedermayer

avcodec/utils: do "calc from frame_bytes, channels, and block_align" in 64bit

Fixes: signed integer overflow: 104962766 * 32 cannot be represented in type 'int'
Fixes: 33614/clusterfuzz-testcase-minimized-ffmpeg_dem_RSD_fuzzer-6252129036664832

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
(cherry picked from commit 3447979d08d701581a65f7275425cb1a59302319)
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=a21c64199cddbea0356c98afa756fcf158634fd8
---

 libavcodec/utils.c | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/libavcodec/utils.c b/libavcodec/utils.c
index d678355d3c..3f69c9c114 100644
--- a/libavcodec/utils.c
+++ b/libavcodec/utils.c
@@ -783,25 +783,33 @@ static int get_audio_frame_duration(enum AVCodecID id, int sr, int ch, int ba,
             if (ba > 0) {
                 /* calc from frame_bytes, channels, and block_align */
                 int blocks = frame_bytes / ba;
-                int64_t tmp;
+                int64_t tmp = 0;
                 switch (id) {
                 case AV_CODEC_ID_ADPCM_IMA_WAV:
                     if (bps < 2 || bps > 5)
                         return 0;
                     tmp = blocks * (1LL + (ba - 4 * ch) / (bps * ch) * 8);
-                    if (tmp != (int)tmp)
-                        return 0;
-                    return tmp;
+                    break;
                 case AV_CODEC_ID_ADPCM_IMA_DK3:
-                    return blocks * (((ba - 16) * 2 / 3 * 4) / ch);
+                    tmp = blocks * (((ba - 16LL) * 2 / 3 * 4) / ch);
+                    break;
                 case AV_CODEC_ID_ADPCM_IMA_DK4:
-                    return blocks * (1 + (ba - 4 * ch) * 2 / ch);
+                    tmp = blocks * (1 + (ba - 4LL * ch) * 2 / ch);
+                    break;
                 case AV_CODEC_ID_ADPCM_IMA_RAD:
-                    return blocks * ((ba - 4 * ch) * 2 / ch);
+                    tmp = blocks * ((ba - 4LL * ch) * 2 / ch);
+                    break;
                 case AV_CODEC_ID_ADPCM_MS:
-                    return blocks * (2 + (ba - 7 * ch) * 2LL / ch);
+                    tmp = blocks * (2 + (ba - 7LL * ch) * 2LL / ch);
+                    break;
                 case AV_CODEC_ID_ADPCM_MTAF:
-                    return blocks * (ba - 16) * 2 / ch;
+                    tmp = blocks * (ba - 16LL) * 2 / ch;
+                    break;
+                }
+                if (tmp) {
+                    if (tmp != (int)tmp)
+                        return 0;
+                    return tmp;
                 }
             }