[FFmpeg-cvslog] pcmdec: fix output buffer size check by calculating the actual output size

[Justin Ruggles]

ffmpeg | branch: master | Justin Ruggles <justin.ruggles at gmail.com> | Thu Sep 29 16:44:50 2011 -0400| [f1901180e02a766edbc74b8fb8cfbb88b79ef347] | committer: Justin Ruggles

pcmdec: fix output buffer size check by calculating the actual output size
prior to decoding.

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=f1901180e02a766edbc74b8fb8cfbb88b79ef347
---

 libavcodec/pcm.c |   15 +++++++++++----
 1 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/libavcodec/pcm.c b/libavcodec/pcm.c
index 8d93992..c9eb543 100644
--- a/libavcodec/pcm.c
+++ b/libavcodec/pcm.c
@@ -250,7 +250,7 @@ static int pcm_decode_frame(AVCodecContext *avctx,
     const uint8_t *src = avpkt->data;
     int buf_size = avpkt->size;
     PCMDecode *s = avctx->priv_data;
-    int sample_size, c, n;
+    int sample_size, c, n, out_size;
     uint8_t *samples;
     int32_t *dst_int32_t;
 
@@ -286,10 +286,17 @@ static int pcm_decode_frame(AVCodecContext *avctx,
             buf_size -= buf_size % n;
     }
 
-    buf_size= FFMIN(buf_size, *data_size/2);
-
     n = buf_size/sample_size;
 
+    out_size = n * av_get_bytes_per_sample(avctx->sample_fmt);
+    if (avctx->codec_id == CODEC_ID_PCM_DVD ||
+        avctx->codec_id == CODEC_ID_PCM_LXF)
+        out_size *= 2;
+    if (*data_size < out_size) {
+        av_log(avctx, AV_LOG_ERROR, "output buffer too small\n");
+        return AVERROR(EINVAL);
+    }
+
     switch(avctx->codec->id) {
     case CODEC_ID_PCM_U32LE:
         DECODE(32, le32, src, samples, n, 0, 0x80000000)
@@ -450,7 +457,7 @@ static int pcm_decode_frame(AVCodecContext *avctx,
     default:
         return -1;
     }
-    *data_size = samples - (uint8_t *)data;
+    *data_size = out_size;
     return buf_size;
 }