[FFmpeg-cvslog] proresenc: Realloc if buffer is too small

[Christophe Gisquet]

ffmpeg | branch: release/2.2 | Christophe Gisquet <christophe.gisquet at gmail.com> | Mon Aug 18 14:15:23 2014 +0000| [a437298de55c6a6a4f06b12335b3891bf4459082] | committer: Luca Barbato

proresenc: Realloc if buffer is too small

The buffer allocation may be incorrect (e.g. with an alpha plane),
and currently causes the buffer to be set to NULL by init_put_bits,
causing a crash later on.

So, detect that situation, and if detected, reallocate the buffer
and ask for a sample that shows the problem.

CC: libav-stable at libav.org

Signed-off-by: Diego Biurrun <diego at biurrun.de>
Signed-off-by: Luca Barbato <lu_zero at gentoo.org>
(cherry picked from commit 45ce880a9b3e50cfa088f111dffaf8685bd7bc6b)
Signed-off-by: Luca Barbato <lu_zero at gentoo.org>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=a437298de55c6a6a4f06b12335b3891bf4459082
---

 libavcodec/proresenc.c |   38 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 37 insertions(+), 1 deletion(-)

diff --git a/libavcodec/proresenc.c b/libavcodec/proresenc.c
index 90f7406..570a73c 100644
--- a/libavcodec/proresenc.c
+++ b/libavcodec/proresenc.c
@@ -205,6 +205,7 @@ typedef struct ProresContext {
     int bits_per_mb;
     int force_quant;
     int alpha_bits;
+    int warn;
 
     char *vendor;
     int quant_sel;
@@ -933,7 +934,7 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *pkt,
     int sizes[4] = { 0 };
     int slice_hdr_size = 2 + 2 * (ctx->num_planes - 1);
     int frame_size, picture_size, slice_size;
-    int pkt_size, ret;
+    int pkt_size, ret, max_slice_size = 0;
     uint8_t frame_flags;
 
     *avctx->coded_frame           = *pic;
@@ -1020,6 +1021,39 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *pkt,
                 bytestream_put_byte(&buf, slice_hdr_size << 3);
                 slice_hdr = buf;
                 buf += slice_hdr_size - 1;
+                if (pkt_size <= buf - orig_buf + 2 * max_slice_size) {
+                    uint8_t *start = pkt->data;
+                    // Recompute new size according to max_slice_size
+                    // and deduce delta
+                    int delta = 200 + ctx->pictures_per_frame *
+                                ctx->slices_per_picture * max_slice_size -
+                                pkt_size;
+
+                    delta = FFMAX(delta, 2 * max_slice_size);
+                    ctx->frame_size_upper_bound += delta;
+
+                    if (!ctx->warn) {
+                        avpriv_request_sample(avctx,
+                                              "Packet too small: is %i,"
+                                              " needs %i (slice: %i). "
+                                              "Correct allocation",
+                                              pkt_size, delta, max_slice_size);
+                        ctx->warn = 1;
+                    }
+
+                    ret = av_grow_packet(pkt, delta);
+                    if (ret < 0)
+                        return ret;
+
+                    pkt_size += delta;
+                    // restore pointers
+                    orig_buf         = pkt->data + (orig_buf         - start);
+                    buf              = pkt->data + (buf              - start);
+                    picture_size_pos = pkt->data + (picture_size_pos - start);
+                    slice_sizes      = pkt->data + (slice_sizes      - start);
+                    slice_hdr        = pkt->data + (slice_hdr        - start);
+                    tmp              = pkt->data + (tmp              - start);
+                }
                 init_put_bits(&pb, buf, (pkt_size - (buf - orig_buf)) * 8);
                 ret = encode_slice(avctx, pic, &pb, sizes, x, y, q,
                                    mbs_per_slice);
@@ -1034,6 +1068,8 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *pkt,
                 }
                 bytestream_put_be16(&slice_sizes, slice_size);
                 buf += slice_size - slice_hdr_size;
+                if (max_slice_size < slice_size)
+                    max_slice_size = slice_size;
             }
         }