[FFmpeg-cvslog] fraps: release reference buffer on pix_fmt change.

[Ronald S. Bultje]

ffmpeg | branch: release/0.8 | Ronald S. Bultje <rsbultje at gmail.com> | Fri Feb 24 14:11:04 2012 -0800| [678737c26be4fb93dc6b3ede04b8231bd39b2e3b] | committer: Reinhard Tartler

fraps: release reference buffer on pix_fmt change.

Prevents crash when trying to copy from a non-existing plane in e.g.
a RGB32 reference image to a YUV420P target image

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable at libav.org
(cherry picked from commit 830f70442a87a31f7c75565e9380e3caf8333b8a)

Signed-off-by: Anton Khirnov <anton at khirnov.net>
Signed-off-by: Reinhard Tartler <siretart at tauware.de>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=678737c26be4fb93dc6b3ede04b8231bd39b2e3b
---

 libavcodec/fraps.c |   14 +++++++-------
 1 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/libavcodec/fraps.c b/libavcodec/fraps.c
index 2bbc7b9..90989e8 100644
--- a/libavcodec/fraps.c
+++ b/libavcodec/fraps.c
@@ -134,7 +134,7 @@ static int decode_frame(AVCodecContext *avctx,
     uint32_t *luma1,*luma2,*cb,*cr;
     uint32_t offs[4];
     int i, j, is_chroma, planes;
-
+    enum PixelFormat pix_fmt;
 
     header = AV_RL32(buf);
     version = header & 0xff;
@@ -151,12 +151,16 @@ static int decode_frame(AVCodecContext *avctx,
     if (header_size == 8)
         buf+=4;
 
+    pix_fmt = version & 1 ? PIX_FMT_BGR24 : PIX_FMT_YUVJ420P;
+    if (avctx->pix_fmt != pix_fmt && f->data[0]) {
+        avctx->release_buffer(avctx, f);
+    }
+    avctx->pix_fmt = pix_fmt;
+
     switch(version) {
     case 0:
     default:
         /* Fraps v0 is a reordered YUV420 */
-        avctx->pix_fmt = PIX_FMT_YUVJ420P;
-
         if ( (buf_size != avctx->width*avctx->height*3/2+header_size) &&
              (buf_size != header_size) ) {
             av_log(avctx, AV_LOG_ERROR,
@@ -204,8 +208,6 @@ static int decode_frame(AVCodecContext *avctx,
 
     case 1:
         /* Fraps v1 is an upside-down BGR24 */
-        avctx->pix_fmt = PIX_FMT_BGR24;
-
         if ( (buf_size != avctx->width*avctx->height*3+header_size) &&
              (buf_size != header_size) ) {
             av_log(avctx, AV_LOG_ERROR,
@@ -240,7 +242,6 @@ static int decode_frame(AVCodecContext *avctx,
          * Fraps v2 is Huffman-coded YUV420 planes
          * Fraps v4 is virtually the same
          */
-        avctx->pix_fmt = PIX_FMT_YUVJ420P;
         planes = 3;
         f->reference = 1;
         f->buffer_hints = FF_BUFFER_HINTS_VALID |
@@ -283,7 +284,6 @@ static int decode_frame(AVCodecContext *avctx,
     case 3:
     case 5:
         /* Virtually the same as version 4, but is for RGB24 */
-        avctx->pix_fmt = PIX_FMT_BGR24;
         planes = 3;
         f->reference = 1;
         f->buffer_hints = FF_BUFFER_HINTS_VALID |