[FFmpeg-devel] [PATCH v3 2/4] libavutil/frame: avoid UB when getting plane sizes

[Brian Kim]

This uses av_image_fill_plane_sizes instead of av_image_fill_pointers
when we are getting plane sizes to avoid UB from adding offsets to NULL.

Signed-off-by: Brian Kim <bkkim at google.com>
---
 libavutil/frame.c | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/libavutil/frame.c b/libavutil/frame.c
index 9884eae054..3ab1aa3242 100644
--- a/libavutil/frame.c
+++ b/libavutil/frame.c
@@ -212,8 +212,10 @@ void av_frame_free(AVFrame **frame)
 static int get_video_buffer(AVFrame *frame, int align)
 {
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(frame->format);
-    int ret, i, padded_height;
+    int ret, i, padded_height, total_size;
     int plane_padding = FFMAX(16 + 16/*STRIDE_ALIGN*/, align);
+    ptrdiff_t linesizes[4];
+    size_t sizes[4];
 
     if (!desc)
         return AVERROR(EINVAL);
@@ -238,12 +240,22 @@ static int get_video_buffer(AVFrame *frame, int align)
             frame->linesize[i] = FFALIGN(frame->linesize[i], align);
     }
 
+    for (i = 0; i < 4; i++)
+        linesizes[i] = frame->linesize[i];
+
     padded_height = FFALIGN(frame->height, 32);
-    if ((ret = av_image_fill_pointers(frame->data, frame->format, padded_height,
-                                      NULL, frame->linesize)) < 0)
+    if ((ret = av_image_fill_plane_sizes(sizes, frame->format,
+                                         padded_height, linesizes)) < 0)
         return ret;
 
-    frame->buf[0] = av_buffer_alloc(ret + 4*plane_padding);
+    total_size = 4*plane_padding;
+    for (i = 0; i < 4; i++) {
+        if (sizes[i] > INT_MAX - total_size)
+            return AVERROR(EINVAL);
+        total_size += sizes[i];
+    }
+
+    frame->buf[0] = av_buffer_alloc(total_size);
     if (!frame->buf[0]) {
         ret = AVERROR(ENOMEM);
         goto fail;
-- 
2.27.0.389.gc38d7665816-goog