[FFmpeg-cvslog] mp3: properly forward mp_decode_frame errors

[Luca Barbato]

ffmpeg | branch: release/0.10 | Luca Barbato <lu_zero at gentoo.org> | Mon Oct 22 19:11:05 2012 +0200| [a5290800f5716a50ff53761164955be09a4e5581] | committer: Reinhard Tartler

mp3: properly forward mp_decode_frame errors

The function can return either a parsing error or a memory management
error.

Fixes: CVE-2012-2797

(cherry picked from commit 9ab0874ea8b6774c6f5470dba2b5b4615a610d0d)

Conflicts:

	libavcodec/mpegaudiodec.c

Signed-off-by: Reinhard Tartler <siretart at tauware.de>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=a5290800f5716a50ff53761164955be09a4e5581
---

 libavcodec/mpegaudiodec.c |   32 ++++++++++++++++++++------------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/libavcodec/mpegaudiodec.c b/libavcodec/mpegaudiodec.c
index bb1baef..cd0d26d 100644
--- a/libavcodec/mpegaudiodec.c
+++ b/libavcodec/mpegaudiodec.c
@@ -1632,7 +1632,7 @@ static int decode_frame(AVCodecContext * avctx, void *data, int *got_frame_ptr,
     int buf_size        = avpkt->size;
     MPADecodeContext *s = avctx->priv_data;
     uint32_t header;
-    int out_size;
+    int ret;
 
     if (buf_size < HEADER_SIZE)
         return AVERROR_INVALIDDATA;
@@ -1663,21 +1663,22 @@ static int decode_frame(AVCodecContext * avctx, void *data, int *got_frame_ptr,
         buf_size= s->frame_size;
     }
 
-    out_size = mp_decode_frame(s, NULL, buf, buf_size);
-    if (out_size >= 0) {
+    ret = mp_decode_frame(s, NULL, buf, buf_size);
+    if (ret >= 0) {
         *got_frame_ptr   = 1;
         *(AVFrame *)data = s->frame;
         avctx->sample_rate = s->sample_rate;
         //FIXME maybe move the other codec info stuff from above here too
     } else {
         av_log(avctx, AV_LOG_ERROR, "Error while decoding MPEG audio frame.\n");
-        /* Only return an error if the bad frame makes up the whole packet.
-           If there is more data in the packet, just consume the bad frame
-           instead of returning an error, which would discard the whole
-           packet. */
+        /* Only return an error if the bad frame makes up the whole packet or
+         * the error is related to buffer management.
+         * If there is more data in the packet, just consume the bad frame
+         * instead of returning an error, which would discard the whole
+         * packet. */
         *got_frame_ptr = 0;
-        if (buf_size == avpkt->size)
-            return out_size;
+        if (buf_size == avpkt->size || ret != AVERROR_INVALIDDATA)
+            return ret;
     }
     s->frame_size = 0;
     return buf_size;
@@ -1698,7 +1699,7 @@ static int decode_frame_adu(AVCodecContext *avctx, void *data,
     int buf_size        = avpkt->size;
     MPADecodeContext *s = avctx->priv_data;
     uint32_t header;
-    int len, out_size;
+    int len, out_size, ret = 0;
 
     len = buf_size;
 
@@ -1735,7 +1736,11 @@ static int decode_frame_adu(AVCodecContext *avctx, void *data,
         out_size = buf_size;
     else
 #endif
-    out_size = mp_decode_frame(s, NULL, buf, buf_size);
+    ret = mp_decode_frame(s, NULL, buf, buf_size);
+    if (ret < 0) {
+        av_log(avctx, AV_LOG_ERROR, "Error while decoding MPEG audio frame.\n");
+        return ret;
+    }
 
     *got_frame_ptr   = 1;
     *(AVFrame *)data = s->frame;
@@ -1942,7 +1947,10 @@ static int decode_frame_mp3on4(AVCodecContext *avctx, void *data,
         }
         ch += m->nb_channels;
 
-        out_size += mp_decode_frame(m, outptr, buf, fsize);
+        if ((ret = mp_decode_frame(m, outptr, buf, fsize)) < 0)
+            return ret;
+
+        out_size += ret;
         buf      += fsize;
         len      -= fsize;