[FFmpeg-cvslog] Fixed crash in palette handling when converting certain .png images to . pcx or .bmp.

[Frank Vernaillen]

ffmpeg | branch: master | Frank Vernaillen <fr_ve at hotmail.com> | Tue Dec 27 20:47:49 2011 +0100| [ad1c50255735c20bd86572d3e8b3c88a5ca6c8f1] | committer: Michael Niedermayer

Fixed crash in palette handling when converting certain .png images to .pcx or .bmp.

The existing code expected a palette buffer holding 256 uint32_t's allocated in the data[1] field of the AVFrame structure, but data[1] was NULL. The bug is fixed by using a fixed local array (palette256) to hold the palette instead.

This solves http://ffmpeg.org/trac/ffmpeg/ticket/833

Signed-off-by: Frank Vernaillen <fr_ve at hotmail.com>
Signed-off-by: Michael Niedermayer <michaelni at gmx.at>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=ad1c50255735c20bd86572d3e8b3c88a5ca6c8f1
---

 libavcodec/bmpenc.c |    7 ++++++-
 libavcodec/pcxenc.c |    7 +++++++
 2 files changed, 13 insertions(+), 1 deletions(-)

diff --git a/libavcodec/bmpenc.c b/libavcodec/bmpenc.c
index 63c3b72..9bd700b 100644
--- a/libavcodec/bmpenc.c
+++ b/libavcodec/bmpenc.c
@@ -24,6 +24,7 @@
 #include "avcodec.h"
 #include "bytestream.h"
 #include "bmp.h"
+#include <assert.h>
 
 static const uint32_t monoblack_pal[] = { 0x000000, 0xFFFFFF };
 static const uint32_t rgb565_masks[]  = { 0xF800, 0x07E0, 0x001F };
@@ -69,6 +70,7 @@ static int bmp_encode_frame(AVCodecContext *avctx, unsigned char *buf, int buf_s
     AVFrame * const p= (AVFrame*)&s->picture;
     int n_bytes_image, n_bytes_per_row, n_bytes, i, n, hsize;
     const uint32_t *pal = NULL;
+    uint32_t palette256[256];
     int pad_bytes_per_row, pal_entries = 0, compression = BMP_RGB;
     int bit_count = avctx->bits_per_coded_sample;
     uint8_t *ptr;
@@ -87,7 +89,10 @@ static int bmp_encode_frame(AVCodecContext *avctx, unsigned char *buf, int buf_s
     case PIX_FMT_RGB4_BYTE:
     case PIX_FMT_BGR4_BYTE:
     case PIX_FMT_GRAY8:
-        ff_set_systematic_pal2((uint32_t*)p->data[1], avctx->pix_fmt);
+        assert(bit_count == 8);
+        ff_set_systematic_pal2(palette256, avctx->pix_fmt);
+        pal = palette256;
+        break;
     case PIX_FMT_PAL8:
         pal = (uint32_t *)p->data[1];
         break;
diff --git a/libavcodec/pcxenc.c b/libavcodec/pcxenc.c
index 816223e..a39e221 100644
--- a/libavcodec/pcxenc.c
+++ b/libavcodec/pcxenc.c
@@ -28,6 +28,7 @@
 
 #include "avcodec.h"
 #include "bytestream.h"
+#include "libavutil/imgutils.h"
 
 typedef struct PCXContext {
     AVFrame picture;
@@ -105,6 +106,7 @@ static int pcx_encode_frame(AVCodecContext *avctx,
 
     int bpp, nplanes, i, y, line_bytes, written;
     const uint32_t *pal = NULL;
+    uint32_t palette256[256];
     const uint8_t *src;
 
     *pict = *(AVFrame *)data;
@@ -126,6 +128,11 @@ static int pcx_encode_frame(AVCodecContext *avctx,
     case PIX_FMT_RGB4_BYTE:
     case PIX_FMT_BGR4_BYTE:
     case PIX_FMT_GRAY8:
+        bpp = 8;
+        nplanes = 1;
+        ff_set_systematic_pal2(palette256, avctx->pix_fmt);
+        pal = palette256;
+        break;
     case PIX_FMT_PAL8:
         bpp = 8;
         nplanes = 1;