[FFmpeg-cvslog] ac3dec: validate channel output mode against channel count

[Justin Ruggles]

ffmpeg | branch: release/0.10 | Justin Ruggles <justin.ruggles at gmail.com> | Wed Feb 20 11:41:20 2013 -0500| [763519536b63636006c9a421a4b83a58d353b84e] | committer: Luca Barbato

ac3dec: validate channel output mode against channel count

Damaged frames can lead to a mismatch, which can cause a segfault
due to using an incorrect channel mapping.

CC:libav-stable at libav.org
(cherry picked from commit d7c450436fcb9d3ecf59884a574e7684183e753d)

Conflicts:

	libavcodec/ac3dec.c

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=763519536b63636006c9a421a4b83a58d353b84e
---

 libavcodec/ac3dec.c |   14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/libavcodec/ac3dec.c b/libavcodec/ac3dec.c
index 28a783a..61097e9 100644
--- a/libavcodec/ac3dec.c
+++ b/libavcodec/ac3dec.c
@@ -1373,8 +1373,10 @@ static int ac3_decode_frame(AVCodecContext * avctx, void *data,
     if (!err) {
         avctx->sample_rate = s->sample_rate;
         avctx->bit_rate    = s->bit_rate;
+    }
 
-        /* channel config */
+    /* channel config */
+    if (!err || (s->channels && s->out_channels != s->channels)) {
         s->out_channels = s->channels;
         s->output_mode  = s->channel_mode;
         if (s->lfe_on)
@@ -1393,18 +1395,18 @@ static int ac3_decode_frame(AVCodecContext * avctx, void *data,
                 s->fbw_channels == s->out_channels)) {
             set_downmix_coeffs(s);
         }
-    } else if (!s->out_channels) {
-        s->out_channels = avctx->channels;
-        if (s->out_channels < s->channels)
-            s->output_mode  = s->out_channels == 1 ? AC3_CHMODE_MONO : AC3_CHMODE_STEREO;
+    } else if (!s->channels) {
+        av_log(avctx, AV_LOG_ERROR, "unable to determine channel mode\n");
+        return AVERROR_INVALIDDATA;
     }
+    avctx->channels = s->out_channels;
+
     /* set audio service type based on bitstream mode for AC-3 */
     avctx->audio_service_type = s->bitstream_mode;
     if (s->bitstream_mode == 0x7 && s->channels > 1)
         avctx->audio_service_type = AV_AUDIO_SERVICE_TYPE_KARAOKE;
 
     /* get output buffer */
-    avctx->channels = s->out_channels;
     s->frame.nb_samples = s->num_blocks * 256;
     if ((ret = avctx->get_buffer(avctx, &s->frame)) < 0) {
         av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n");