[FFmpeg-cvslog] Fix a buffer overflow in libx264 interface to x264 encoder. Previous code ignored the compressed buffer size passed in. This change returns as many complete NALs as can fit in the buffer, and logs an error message.

[Mike Scheutzow]

ffmpeg | branch: master | Mike Scheutzow <mike.scheutzow at alcatel-lucent.com> | Mon Sep 26 10:57:53 2011 -0400| [e2dae1faa84ada5746ac2114de7eb68abd824131] | committer: Michael Niedermayer

Fix a buffer overflow in libx264 interface to x264 encoder. Previous code ignored the compressed buffer size passed in. This change returns as many complete NALs as can fit in the buffer, and logs an error message.

Signed-off-by: Mike Scheutzow <mike.scheutzow at alcatel-lucent.com>
Signed-off-by: Michael Niedermayer <michaelni at gmx.at>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=e2dae1faa84ada5746ac2114de7eb68abd824131
---

 libavcodec/libx264.c |   10 ++++++++++
 1 files changed, 10 insertions(+), 0 deletions(-)

diff --git a/libavcodec/libx264.c b/libavcodec/libx264.c
index 8732672..a089206 100644
--- a/libavcodec/libx264.c
+++ b/libavcodec/libx264.c
@@ -96,9 +96,14 @@ static int encode_nals(AVCodecContext *ctx, uint8_t *buf, int size,
 
     /* Write the SEI as part of the first frame. */
     if (x4->sei_size > 0 && nnal > 0) {
+        if (x4->sei_size > size) {
+            av_log(ctx, AV_LOG_ERROR, "Error: nal buffer is too small\n");
+            return -1;
+        }
         memcpy(p, x4->sei, x4->sei_size);
         p += x4->sei_size;
         x4->sei_size = 0;
+        // why is x4->sei not freed?
     }
 
     for (i = 0; i < nnal; i++){
@@ -109,6 +114,11 @@ static int encode_nals(AVCodecContext *ctx, uint8_t *buf, int size,
             memcpy(x4->sei, nals[i].p_payload, nals[i].i_payload);
             continue;
         }
+        if (nals[i].i_payload > (size - (p - buf))) {
+            // return only complete nals which fit in buf
+            av_log(ctx, AV_LOG_ERROR, "Error: nal buffer is too small\n");
+            break;
+        }
         memcpy(p, nals[i].p_payload, nals[i].i_payload);
         p += nals[i].i_payload;
     }