[FFmpeg-cvslog] apedec: check output buffer size after calculating actual output size

[Justin Ruggles]

ffmpeg | branch: master | Justin Ruggles <justin.ruggles at gmail.com> | Mon Oct 31 15:03:14 2011 -0400| [4315c7d35aa946fb3a0da9a30f08fb4e0ca8edfb] | committer: Justin Ruggles

apedec: check output buffer size after calculating actual output size

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=4315c7d35aa946fb3a0da9a30f08fb4e0ca8edfb
---

 libavcodec/apedec.c |   18 ++++++++++--------
 1 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/libavcodec/apedec.c b/libavcodec/apedec.c
index 0619358..1567025 100644
--- a/libavcodec/apedec.c
+++ b/libavcodec/apedec.c
@@ -816,15 +816,9 @@ static int ape_decode_frame(AVCodecContext *avctx,
     int16_t *samples = data;
     uint32_t nblocks;
     int i;
-    int blockstodecode;
+    int blockstodecode, out_size;
     int bytes_used;
 
-    /* should not happen but who knows */
-    if (BLOCKS_PER_LOOP * 2 * avctx->channels > *data_size) {
-        av_log (avctx, AV_LOG_ERROR, "Output buffer is too small.\n");
-        return AVERROR(EINVAL);
-    }
-
     /* this should never be negative, but bad things will happen if it is, so
        check it just to make sure. */
     av_assert0(s->samples >= 0);
@@ -883,6 +877,13 @@ static int ape_decode_frame(AVCodecContext *avctx,
     nblocks = s->samples;
     blockstodecode = FFMIN(BLOCKS_PER_LOOP, nblocks);
 
+    out_size = blockstodecode * avctx->channels *
+               av_get_bytes_per_sample(avctx->sample_fmt);
+    if (*data_size < out_size) {
+        av_log(avctx, AV_LOG_ERROR, "Output buffer is too small.\n");
+        return AVERROR(EINVAL);
+    }
+
     s->error=0;
 
     if ((s->channels == 1) || (s->frameflags & APE_FRAMECODE_PSEUDO_STEREO))
@@ -905,9 +906,10 @@ static int ape_decode_frame(AVCodecContext *avctx,
 
     s->samples -= blockstodecode;
 
-    *data_size = blockstodecode * 2 * s->channels;
     bytes_used = s->samples ? s->ptr - s->last_ptr : buf_size;
     s->last_ptr = s->ptr;
+
+    *data_size = out_size;
     return bytes_used;
 }