[FFmpeg-cvslog] avcodec/wmalosslessdec: ensure channel count in the private context and decoder context are consistent

[James Almer]

ffmpeg | branch: master | James Almer <jamrial at gmail.com> | Fri Mar 18 09:46:09 2022 -0300| [7c35aa60a5131b129cf474ae7f27f949a26ae21c] | committer: James Almer

avcodec/wmalosslessdec: ensure channel count in the private context and decoder context are consistent

Fixes: Out of array write
Fixes: 45613/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_WMALOSSLESS_fuzzer-4539073606320128
Fixes: 46008/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_WMALOSSLESS_fuzzer-4681245747970048

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg

Reviewed-by: Michael Niedermayer <michael at niedermayer.cc>
Signed-off-by: James Almer <jamrial at gmail.com>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=7c35aa60a5131b129cf474ae7f27f949a26ae21c
---

 libavcodec/wmalosslessdec.c | 49 ++++++++++++++++++++++++---------------------
 1 file changed, 26 insertions(+), 23 deletions(-)

diff --git a/libavcodec/wmalosslessdec.c b/libavcodec/wmalosslessdec.c
index 7d5753dccd..d4f3462804 100644
--- a/libavcodec/wmalosslessdec.c
+++ b/libavcodec/wmalosslessdec.c
@@ -198,15 +198,6 @@ static av_cold int decode_init(AVCodecContext *avctx)
         return AVERROR_PATCHWELCOME;
     }
 
-    s->max_frame_size = MAX_FRAMESIZE * avctx->ch_layout.nb_channels;
-    s->frame_data = av_mallocz(s->max_frame_size + AV_INPUT_BUFFER_PADDING_SIZE);
-    if (!s->frame_data)
-        return AVERROR(ENOMEM);
-
-    s->avctx = avctx;
-    ff_llauddsp_init(&s->dsp);
-    init_put_bits(&s->pb, s->frame_data, s->max_frame_size);
-
     if (avctx->extradata_size >= 18) {
         s->decode_flags    = AV_RL16(edata_ptr + 14);
         channel_mask       = AV_RL32(edata_ptr +  2);
@@ -231,6 +222,32 @@ static av_cold int decode_init(AVCodecContext *avctx)
         return AVERROR_PATCHWELCOME;
     }
 
+    if (channel_mask) {
+        av_channel_layout_uninit(&avctx->ch_layout);
+        av_channel_layout_from_mask(&avctx->ch_layout, channel_mask);
+    }
+
+    s->num_channels = avctx->ch_layout.nb_channels;
+
+    /* extract lfe channel position */
+    s->lfe_channel = -1;
+
+    if (channel_mask & 8) {
+        unsigned int mask;
+        for (mask = 1; mask < 16; mask <<= 1)
+            if (channel_mask & mask)
+                ++s->lfe_channel;
+    }
+
+    s->max_frame_size = MAX_FRAMESIZE * avctx->ch_layout.nb_channels;
+    s->frame_data = av_mallocz(s->max_frame_size + AV_INPUT_BUFFER_PADDING_SIZE);
+    if (!s->frame_data)
+        return AVERROR(ENOMEM);
+
+    s->avctx = avctx;
+    ff_llauddsp_init(&s->dsp);
+    init_put_bits(&s->pb, s->frame_data, s->max_frame_size);
+
     /* generic init */
     s->log2_frame_size = av_log2(avctx->block_align) + 4;
 
@@ -264,24 +281,10 @@ static av_cold int decode_init(AVCodecContext *avctx)
         return AVERROR_INVALIDDATA;
     }
 
-    s->num_channels = avctx->ch_layout.nb_channels;
-
-    /* extract lfe channel position */
-    s->lfe_channel = -1;
-
-    if (channel_mask & 8) {
-        unsigned int mask;
-        for (mask = 1; mask < 16; mask <<= 1)
-            if (channel_mask & mask)
-                ++s->lfe_channel;
-    }
-
     s->frame = av_frame_alloc();
     if (!s->frame)
         return AVERROR(ENOMEM);
 
-    av_channel_layout_uninit(&avctx->ch_layout);
-    av_channel_layout_from_mask(&avctx->ch_layout, channel_mask);
     return 0;
 }